Commit 532d2c1 for stable-diffusion.cpp
commit 532d2c15c430a785a712b2d94281f04e733008a4
Author: leejet <leejet714@gmail.com>
Date: Sun Nov 15 18:33:53 2026 +0800
refactor(api): adopt opaque options, tensors, and results (#2365)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 98ec75f..59107d4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -240,6 +240,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
"src/*.h"
"src/*.cpp"
"src/*.hpp"
+ "src/api/*.h"
+ "src/api/*.cpp"
"src/conditioning/*.h"
"src/conditioning/*.cpp"
"src/conditioning/*.hpp"
@@ -431,6 +433,9 @@ target_include_directories(${SD_LIB} PUBLIC include)
target_include_directories(${SD_LIB} PRIVATE src src/core)
target_include_directories(${SD_LIB} PUBLIC thirdparty)
target_compile_features(${SD_LIB} PUBLIC c_std_11 cxx_std_17)
+set_target_properties(${SD_LIB} PROPERTIES
+ CXX_VISIBILITY_PRESET hidden
+ VISIBILITY_INLINES_HIDDEN ON)
if (SD_BUILD_EXAMPLES)
diff --git a/docs/README.md b/docs/README.md
index 4e998a5..5490146 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -15,6 +15,7 @@ For the project overview and supported models, see the [project README](../READM
## Generation and inputs
+- [C API and migration](guides/c_api.md)
- [Sampling methods and schedulers](guides/sampling.md)
- [Image editing](guides/edit.md)
- [Images in prompts](guides/prompt_images.md)
@@ -38,6 +39,7 @@ For the project overview and supported models, see the [project README](../READM
## Development
+- [C API internals](development/c_api.md)
- [Model configuration and integration conventions](development/model_config.md)
- [ggml operator extensions](development/ggml_extensions.md)
- [Backend performance probes](development/backend_probe.md)
diff --git a/docs/development/c_api.md b/docs/development/c_api.md
new file mode 100644
index 0000000..58587a6
--- /dev/null
+++ b/docs/development/c_api.md
@@ -0,0 +1,63 @@
+# C API Internals
+
+The public declarations live in [stable-diffusion.h](../../include/stable-diffusion.h).
+See the [C API guide](../guides/c_api.md) for keys, value types and ownership rules.
+
+## Internal modules
+
+| Module | Responsibility |
+| --- | --- |
+| `src/api/options.h/.cpp` | Opaque options, key registration, setter validation and conversion to typed configuration. |
+| `src/config.h/.cpp` | Typed configuration and the library defaults for each configuration group. |
+| `src/api/tensor.h/.cpp` | Opaque tensor handles backed by `sd::Tensor<T>`, reference counting and conversion to private media views. |
+| `src/core/media.h` | Private raw media views used by inference and codecs. |
+| `src/api/result.h/.cpp` | Internal result storage, opaque result accessors, metadata attachment and media ownership. |
+| `src/api/api.h/.cpp` | Context, generation, callback and query entry points, plus thread-local diagnostics and exception-to-status handling. |
+| `src/generation_info.h/.cpp` | Construction of generation metadata JSON. |
+| `src/pipeline/generation.h` | Internal generation entry declarations. |
+| `src/stable-diffusion.cpp` | Internal typed image/video generation and model-specific validation. |
+| `src/core/option_types.h/.cpp` | Internal mode enums, name lookup and parsing. |
+
+Options are validated and converted once at the API boundary. The runtime reads
+typed configuration fields. Internal callers such as ADetailer pass typed
+configuration directly to the internal generation entry; they do not rebuild
+options or call public setters.
+
+The option map alone records whether a key was supplied. Zero, false, empty
+strings and explicit automatic choices remain entries until `unset` removes
+them. Each conversion starts with a fresh `OptConfig`, then overlays supplied
+values and validates combinations. Reusing the output configuration cannot
+retain a previous request's overrides. ADetailer applies its operation-specific
+strength default before the overlay.
+
+Fixed defaults live in the configuration constructors. Internal automatic
+policies (such as model-selected sampler/scheduler and automatic eta/flow shift)
+are resolved by the pipeline after refreshing the model sources. Their internal
+sentinel values describe an automatic policy, not the presence of an input key.
+Do not infer presence by comparing a typed value against its default.
+
+CLI/server inference inputs use `std::optional`; their option builders emit
+only engaged fields. Recipe defaults fill missing startup inputs; request
+fields then override the configured recipe/startup values. Explicit empty
+numeric arrays replace inherited arrays, and JSON `null` clears an optional
+input. Frontend-only settings such as output formats and listen ports keep
+local defaults. Frontend checks cover request syntax, file inputs and server
+resource policy. Parameter ranges are checked by the public setters used by the
+option builders; combinations and model applicability are checked when executing.
+Examples include only public library headers and do not query library defaults.
+Capability defaults describe configured overrides; null means that the library
+selects the value, and result metadata reports effective values. Post-generation
+ADetailer clears inherited dimensions and strength before applying pass overrides,
+so its operation defaults also come from the library.
+
+Result storage owns tensors and metadata. Tensor handles contain a dtype-specific
+`sd::Tensor<T>` and an atomic reference count; options retain handles rather than
+copying tensor payloads. Raw pipeline output buffers are converted to tensors at
+the boundary and released there. Internal producers with an `sd::Tensor<T>` can
+move it directly into a handle. Video results keep per-frame tensors and assemble
+a contiguous video tensor only when requested. Cached assembly is synchronized.
+Preview tensors are independent snapshots and may be retained by callbacks.
+Metadata construction is separate from its attachment to a result.
+
+These headers are private implementation details and are not installed as part
+of the public C API.
diff --git a/docs/guides/backend.md b/docs/guides/backend.md
index cc6545d..f7edddf 100644
--- a/docs/guides/backend.md
+++ b/docs/guides/backend.md
@@ -273,8 +273,8 @@ flags are deprecated and emit a warning, but remain supported when
`--conv-direct` is omitted. Use `--conv-direct auto,diffusion=on,controlnet=on`
and `--conv-direct auto,vae=on`, respectively. An explicit
`--conv-direct` takes precedence over both legacy flags regardless of order.
-Library callers can set the appended `sd_ctx_params_t::conv_direct_mode` field
-to the same string; null or empty selects `auto` while honoring legacy true
+Library callers can set the `conv_direct_mode` context-option key with
+`sd_opt_set_string()` to the same string; unset or empty selects `auto` while honoring legacy true
flags. The context copies the string.
## Flash attention (`--flash-attn on|off|auto`)
@@ -339,9 +339,10 @@ row-split configurations, which can fall back to layer split during initializati
Use `--flash-attn on` and `--flash-attn off,diffusion=on`, respectively.
Both aliases write the same mode string as `--flash-attn`; the last occurrence
of any of these options wins. Library callers
-can set `sd_ctx_params_t::flash_attn_mode` to the same string, including
+can set the `flash_attn_mode` context-option key with `sd_opt_set_string()`
+to the same string, including
`"on"`, `"off"`, `"auto"`, or `"auto,te=off,vae=on"`.
-A null or empty string uses automatic selection unless a
+An unset or empty string uses automatic selection unless a
legacy `flash_attn` or `diffusion_flash_attn` boolean is set. The context copies
the string; no new enum is required.
diff --git a/docs/guides/c_api.md b/docs/guides/c_api.md
new file mode 100644
index 0000000..055056d
--- /dev/null
+++ b/docs/guides/c_api.md
@@ -0,0 +1,507 @@
+# C API
+
+Use purpose-specific opaque options for model loading, generation and
+postprocessing, and
+opaque `sd_result_t` objects to read generated outputs. The library allocates and
+frees these objects; applications must not allocate them by size or access fields.
+
+## Loading and generation
+
+This C example loads a checkpoint and generates an image. Check every setter:
+an unsuccessful setter preserves the previous value, including its unset state.
+
+```c
+#include <stdio.h>
+#include "stable-diffusion.h"
+
+int generate(const char* model_path) {
+ int rc = 1;
+ sd_ctx_t* ctx = NULL;
+ sd_result_t* result = NULL;
+ sd_ctx_opt_t* ctx_opt = sd_ctx_opt_create();
+ sd_img_gen_opt_t* image_opt = NULL;
+ if (!ctx_opt) goto error;
+ if (sd_opt_set_string(ctx_opt, "model_path", model_path) != SD_STATUS_OK ||
+ sd_opt_set_int(ctx_opt, "n_threads", 8) != SD_STATUS_OK ||
+ sd_ctx_create(ctx_opt, &ctx) != SD_STATUS_OK)
+ goto error;
+ sd_opt_free(ctx_opt);
+ ctx_opt = NULL;
+
+ image_opt = sd_img_gen_opt_create();
+ if (!image_opt) goto error;
+ if (sd_opt_set_string(image_opt, "prompt", "a small house") != SD_STATUS_OK ||
+ sd_opt_set_int(image_opt, "width", 512) != SD_STATUS_OK ||
+ sd_opt_set_int(image_opt, "height", 512) != SD_STATUS_OK ||
+ sd_opt_set_int(image_opt, "seed", 42) != SD_STATUS_OK ||
+ sd_opt_set_int(image_opt, "sample.steps", 20) != SD_STATUS_OK ||
+ sd_opt_set_string(image_opt, "sample.method", "euler") != SD_STATUS_OK ||
+ sd_generate_image(ctx, image_opt, &result) != SD_STATUS_OK)
+ goto error;
+
+ for (size_t i = 0; i < sd_result_get_image_count(result); ++i) {
+ const sd_tensor_t* image = sd_result_get_image(result, i);
+ const char* info = sd_result_get_generation_info(result, i);
+ /* Read pixels with sd_tensor_data(image), and consume the UTF-8 JSON info. */
+ (void) image;
+ (void) info;
+ }
+ rc = 0;
+ goto cleanup;
+error:
+ fprintf(stderr, "%s\n", sd_get_last_error());
+cleanup:
+ sd_result_free(result);
+ sd_ctx_free(ctx);
+ sd_opt_free(ctx_opt);
+ sd_opt_free(image_opt);
+ return rc;
+}
+```
+
+Use `sd_generate_video()` with options from `sd_vid_gen_opt_create()` to generate video. Its result
+exposes frames through the same image accessors, optional audio through
+`sd_result_get_audio()`, and effective frame rate through `sd_result_get_fps()`.
+`sd_result_get_kind()` distinguishes image batches from videos. Video generation
+metadata is sequence-wide and is returned for every valid frame index.
+
+All option names are aliases of the same opaque `sd_opt_t` type and share
+all `sd_opt_set_*()`, `sd_opt_unset()` and `sd_opt_free()` functions. The constructor
+permanently binds the object's purpose. Setters reject keys for another purpose
+immediately, and operation entry points reject an option object of the wrong
+purpose even if empty. For example, `prompt` cannot be set on context options,
+and image options cannot be passed to `sd_ctx_create()`. These checks occur at
+runtime; the C typedef aliases provide descriptive names rather than distinct
+compile-time types.
+
+## Operations
+
+Each operation has its own constructor and uses the shared setters and
+`sd_opt_free()`. Creation and execution return `sd_status_t`; output pointers
+are required and set to `NULL` on failure. All outputs use `sd_result_t`.
+
+| Option constructor | Operation |
+| --- | --- |
+| `sd_ctx_opt_create()` | `sd_ctx_create()` |
+| `sd_img_gen_opt_create()` | `sd_generate_image()` |
+| `sd_vid_gen_opt_create()` | `sd_generate_video()` |
+| `sd_upscaler_opt_create()` | `sd_upscaler_create()` |
+| `sd_upscale_opt_create()` | `sd_upscale()` |
+| `sd_vid_upscaler_opt_create()` | `sd_vid_upscaler_create()` |
+| `sd_vid_upscale_opt_create()` | `sd_upscale_video()` |
+| `sd_adetailer_opt_create()` | `sd_adetailer_create()` |
+| `sd_adetail_opt_create()` | `sd_adetail()` |
+
+Image upscaling loads `model_path` with optional `n_threads`, `backend`,
+`params_backend`, `conv_direct` and `tile_size` (default 128). Execution takes
+an RGB `image`. Its scale is fixed by the loaded model and can be queried with
+`sd_upscaler_get_scale()`; metadata records that actual scale. The former
+image-upscale factor argument was unused and has been removed.
+
+```c
+sd_upscaler_opt_t* load = sd_upscaler_opt_create();
+sd_upscaler_t* upscaler = NULL;
+sd_upscale_opt_t* opt = NULL;
+sd_result_t* result = NULL;
+sd_status_t status = load ? sd_opt_set_string(load, "model_path", model_path)
+ : SD_STATUS_OUT_OF_MEMORY;
+if (status == SD_STATUS_OK) status = sd_upscaler_create(load, &upscaler);
+if (status == SD_STATUS_OK) {
+ opt = sd_upscale_opt_create();
+ status = sd_opt_set_tensor(opt, "image", image);
+ if (status == SD_STATUS_OK) status = sd_upscale(upscaler, opt, &result);
+}
+/* Consume result on success. */
+sd_result_free(result);
+sd_opt_free(opt);
+sd_upscaler_free(upscaler);
+sd_opt_free(load);
+```
+
+Video-upscaler loading requires `diffusion_model_path`, `vae_path` and
+`upscale_factor`; backend/thread/attention settings are optional. The scale must
+match both checkpoints (RelayVSR uses 2 or 4). Execution takes RGB `frames` with
+equal dimensions. Defaults are FPS 24, seed 42, RelayVSR `window_size` 16, and
+SeedVR2 `temporal_chunk_size` 5. Free the model with `sd_vid_upscaler_free()`.
+
+ADetailer loading takes the detector `model_path` and optional backend/thread
+settings. Call `sd_adetail(adetailer, diffusion_ctx, opt, &result)` with an
+`sd_adetail_opt_t`. Set `image` for the source image, and reuse the applicable
+image-generation keys for inpainting: `prompt`, `negative_prompt`, dimensions,
+strength, seed, sampling, LoRAs, cache, VAE tiling and circular axes. Inpainting
+defaults to 512×512 and strength 0.4. `adetail.prompt` and
+`adetail.negative_prompt` supply region prompt overrides; `adetail.args` accepts
+the existing detector/mask/inpaint argument string documented by the CLI
+`--extra-ad-args` option. Reference/control images, hires and batching are not
+ADetailer options. Free the detector with `sd_adetailer_free()`.
+
+## String-valued modes
+
+Sampler, scheduler, prediction, RNG, cache, control, hires and VAE option enums
+are internal. Set their documented string values through `sd_opt_set_string()`.
+Leave `sample.method` and `sample.scheduler` unset, or set them to `"auto"`,
+to select defaults for the loaded model and sampler.
+
+The bundled CLI and server retain these modes as strings and pass them directly
+to setters. Enum parsers, enum name helpers, parameter configuration layouts and
+name tables are internal and are not exported by the shared library.
+
+Preview registration also uses strings:
+
+```c
+sd_status_t status = sd_set_preview_callback(callback, "proj", 5,
+ true, false, user_data);
+/* Disable registration when the context is idle. */
+sd_set_preview_callback(NULL, "none", 1, false, false, NULL);
+```
+
+Preview modes are `none`, `proj`, `tae` and `vae`. A null or unknown mode returns
+`SD_STATUS_INVALID_ARGUMENT` and preserves the previous registration. See the
+public header for interval, noisy/denoised and callback lifetime semantics.
+
+## Tensor data
+
+All media payloads use opaque `sd_tensor_t` handles, backed by the library's
+`sd::Tensor<T>`. The declarations are in `stable-diffusion.h`; no additional
+public header is required. Dimensions are ordered from most contiguous to least
+contiguous, matching `sd::Tensor`:
+
+| Media | Element type | Shape |
+| --- | --- | --- |
+| Image | `SD_TENSOR_U8` | `[channels, width, height]` |
+| Audio samples | `SD_TENSOR_F32` | `[channels, samples]` |
+| Video frames | `SD_TENSOR_U8` | `[channels, width, height, frames]` |
+
+Channels are interleaved. Audio duration is `samples / sample_rate`; sample count
+and channel count come from the tensor dimensions. `sd_audio_t` adds `sample_rate`
+to a sample tensor. `sd_video_t` adds FPS and an optional audio descriptor to a
+video tensor. Image dimensions and channel count are entirely in the tensor.
+
+`sd_tensor_create(type, rank, shape, data, data_size)` copies the supplied bytes.
+The byte count must exactly match the element type and dimensions. General
+tensors support U8, I32, I64, F32 and F64, rank up to 8, rank-zero scalars and
+zero-sized dimensions. Media keys validate their required type, rank and positive
+dimensions.
+
+```c
+int64_t shape[] = {3, width, height};
+sd_tensor_t* image = sd_tensor_create(SD_TENSOR_U8, 3, shape,
+ pixels, (size_t)width * height * 3);
+if (image) {
+ sd_status_t status = sd_opt_set_tensor(opt, "init_image", image);
+ sd_tensor_free(image); /* opt retains the tensor after a successful setter. */
+ (void)status;
+}
+```
+
+Use `sd_opt_set_tensors()` for image lists such as `ref_images`; individual images
+may have different dimensions. Use one 4D tensor for `control_frames` and video
+upscaler `frames`. Use `sd_opt_set_audios()` and `sd_opt_set_videos()` for lists
+whose items also carry sample-rate/FPS metadata. These setters retain all nested
+tensors. Freeing the caller's tensor handles or descriptor arrays does not
+invalidate configured options.
+
+`sd_result_get_image()` returns a U8 image/frame tensor. `sd_result_get_audio()`
+returns sample-rate metadata and a F32 sample tensor. `sd_result_get_video()`
+returns FPS, optional audio and a contiguous 4D tensor, assembled and cached on
+first access. Reading individual video frames avoids that assembly and copy.
+
+Tensor data is read-only. `sd_tensor_retain()` extends the lifetime of any tensor,
+including result and preview tensors, independently of its original owner;
+release each owned reference with `sd_tensor_free()`. Retain/free accept NULL and
+preserve the last diagnostic. Preview callbacks receive an array of borrowed
+image tensors. Prompt-image callbacks return a borrowed image tensor that stays
+valid until the next callback. `sd_preprocess_canny()` returns a new image tensor
+and leaves its input unchanged.
+
+## Values, validation and ownership
+
+- Unset keys retain existing model-loading and generation defaults.
+ `sd_opt_unset()` restores this state and succeeds for an applicable known absent key.
+ Zero, false, empty strings and empty arrays remain explicit values, subject to
+ that key's validation. `NULL` strings are invalid; use `unset` for defaults.
+- Keys and enum values are case-sensitive. Unknown keys return
+ `SD_STATUS_UNKNOWN_KEY`; incorrect setter types return `SD_STATUS_TYPE_MISMATCH`.
+ Numeric overflow/range errors and invalid enum names are rejected by setters.
+ No automatic conversion occurs between integer, floating-point and boolean values.
+- Setters and unset reject keys for another purpose with `SD_STATUS_UNSUPPORTED`,
+ even when set to a default-looking value. Loading/generation checks the option
+ object purpose and parameter combinations. Model-dependent features are checked at generation:
+ for example UCache requires UNet and EasyCache/CacheDIT require DiT. Known
+ high-noise sampling settings and `moe_boundary` are ignored without a high-noise
+ stage; `vace_strength` is ignored by non-VACE models. Actual control-frame inputs
+ still require VACE. Existing size/frame alignment and model-dependent automatic
+ defaults are retained.
+- Scalar strings, numeric arrays, LoRA paths and embedding names/paths are copied.
+ Tensor arrays and audio/video metadata are copied, and tensor references are
+ retained. Empty arrays accept a `NULL` pointer and count 0.
+- Successful context creation owns the configuration it retains. Free or reuse
+ the loading params immediately afterwards. Generation params may be freed or
+ changed after the generation call returns. Do not mutate/free an option object
+ while another operation is using it.
+- Results own their media and metadata independently of the context and params.
+ Read-only views remain valid until `sd_result_free()`. Do not free or modify
+ buffers through a view. Image results have no audio and return FPS 0. Invalid
+ indices return `NULL` with a diagnostic. All new free functions accept `NULL`.
+- Output pointers are required and set to `NULL` on failure. Partial batch
+ cancellation can return successful completed outputs; full cancellation returns
+ `SD_STATUS_CANCELLED` without a result. Other failures distinguish invalid
+ arguments, unsupported features, memory exhaustion, loading and generation.
+- `sd_get_last_error()` returns a thread-local diagnostic. Status/accessor calls
+ clear it on success; free functions preserve it. Read it before another API call.
+
+Option and tensor objects have no public field-layout contract. Future settings
+normally add a key while retaining existing key names, types and meanings. To
+keep an output after releasing the result, retain its tensor first; release the
+retained handle with `sd_tensor_free()` when finished. Result metadata and media
+metadata descriptors remain borrowed from the result and should be copied when
+needed independently.
+
+## Keys
+
+Types below correspond to `sd_opt_set_<type>()`. `context`, `image` and `video`
+refer to model loading and image/video generation. Other operation labels match
+the corresponding option constructor, such as `vid_upscale` for
+`sd_vid_upscale_opt_create()`.
+Array lengths are supplied to setters; there are no separately writable count keys.
+
+Top-level keys retain their existing parameter meanings. Nested names map as follows:
+
+| Key prefix | Configuration |
+| --- | --- |
+| `sample.*` | Base sampling settings; `method`, `steps`, `sigmas`, `args` select the sampler, step count, custom schedule and extra arguments. |
+| `high_noise_sample.*` | High-noise sampling settings for two-stage video models. |
+| `sample.guidance.*` | Text/image CFG, distilled guidance and SLG settings. |
+| `cache.*` | [Inference cache](caching.md) settings. |
+| `hires.*` | High-resolution second pass and upscaling settings. |
+| `vae_tiling.*` | Spatial/temporal VAE tiling settings. |
+| `photomaker.*` | PhotoMaker identity images/embedding and style strength. |
+| `pulid.*` | PuLID identity embedding path and weight. |
+| `image_preprocess.rules` | [Image preprocessing](image_preprocessing.md) rule string. |
+
+The library supplies inference defaults. CLI/server inputs that are omitted do
+not override them. Model recipes may supply presets; explicit startup options
+and request fields take priority over recipe values. A supplied value equal to
+a default is still explicit and undergoes model applicability checks.
+
+Generation defaults include 512×512, a random seed (`-1`), 20 sampling steps,
+model-selected sampler/scheduler, text CFG 7, distilled guidance 3.5, strength
+0.75, and disabled caching/hires/tiling. Image batches default to 1; videos default
+to 6 requested frames at 16 FPS, subject to model alignment. Omitted eta, flow shift
+and image CFG retain automatic defaults; do not pass infinity to request defaults.
+Use `unset`. High-noise steps default to `-1` (automatic allocation). The high-noise pass
+shares the base scheduler, sigmas, flow shift and timestep shifting; set those
+through `sample.*`. Separate high-noise keys for these settings are not supported.
+
+Sampler/scheduler values follow the [sampling guide](sampling.md); `"auto"`
+requests automatic selection. RNG values are `std_default`, `cuda`, `cpu`, `auto`.
+Prediction modes are `auto`, `eps`, `v`, `edm_v`, `sd3_flow`, `flux_flow`,
+`sefi_flow`, `minit2i_flow`, `sensenova_u1_flow`, `flux3_flow`, `h3_av_flow`,
+`ming_image_flow`. Weight types use the names returned by `sd_type_name()`,
+plus `auto`. VAE formats are `auto`, `flux`, `sd3`,
+`flux2`, `wan`. LoRA apply modes are `auto`, `immediately`, `at_runtime`.
+Cache modes are `disabled`, `easycache`, `ucache`, `dbcache`, `taylorseer`,
+`cache_dit`, `spectrum`. Control types are `none`, `openpose`, `depth`, `scribble`, `canny`, `normal`,
+`segment`, `tile`, `repaint`, with the existing aliases such as `hed` and `lineart`. Hires upscalers are `none`, `latent`,
+`latent_nearest`, `latent_nearest_exact`, `latent_antialiased`, `latent_bicubic`,
+`latent_bicubic_antialiased`, `lanczos`, `nearest`, `model`.
+
+Integers must fit the underlying field (32 bits except `seed`, which is signed
+64-bit). Dimensions, batch size, video frames, FPS and base steps are positive.
+Floating-point settings must be finite and fit float32; strength, denoising
+strength, percent/range settings and MOE boundary lie in [0, 1]. Custom sigma
+arrays are empty or strictly decreasing nonnegative finite values ending in zero,
+with at least two entries. SLG layer indices are nonnegative integers.
+
+| Key | Setter type | Operation |
+| --- | --- | --- |
+| `adetail.args` | `string` | adetail |
+| `adetail.negative_prompt` | `string` | adetail |
+| `adetail.prompt` | `string` | adetail |
+| `attn_scale` | `float` | context |
+| `audio_encoder_path` | `string` | context |
+| `audio_vae_path` | `string` | context |
+| `auto_fit` | `bool` | context |
+| `backend` | `string` | context, upscaler, vid_upscaler, adetailer |
+| `batch_count` | `int` | image |
+| `batched_cfg` | `bool` | context |
+| `cache.bn_compute_blocks` | `int` | image, video, adetail |
+| `cache.end_percent` | `float` | image, video, adetail |
+| `cache.error_decay_rate` | `float` | image, video, adetail |
+| `cache.fn_compute_blocks` | `int` | image, video, adetail |
+| `cache.max_cached_steps` | `int` | image, video, adetail |
+| `cache.max_continuous_cached_steps` | `int` | image, video, adetail |
+| `cache.max_warmup_steps` | `int` | image, video, adetail |
+| `cache.mode` | `string` | image, video, adetail |
+| `cache.reset_error_on_compute` | `bool` | image, video, adetail |
+| `cache.residual_diff_threshold` | `float` | image, video, adetail |
+| `cache.reuse_threshold` | `float` | image, video, adetail |
+| `cache.scm_mask` | `string` | image, video, adetail |
+| `cache.scm_policy_dynamic` | `bool` | image, video, adetail |
+| `cache.spectrum_flex_window` | `float` | image, video, adetail |
+| `cache.spectrum_lam` | `float` | image, video, adetail |
+| `cache.spectrum_m` | `int` | image, video, adetail |
+| `cache.spectrum_stop_percent` | `float` | image, video, adetail |
+| `cache.spectrum_w` | `float` | image, video, adetail |
+| `cache.spectrum_warmup_steps` | `int` | image, video, adetail |
+| `cache.spectrum_window_size` | `int` | image, video, adetail |
+| `cache.start_percent` | `float` | image, video, adetail |
+| `cache.taylorseer_n_derivatives` | `int` | image, video, adetail |
+| `cache.taylorseer_skip_interval` | `int` | image, video, adetail |
+| `cache.use_relative_threshold` | `bool` | image, video, adetail |
+| `circular_x` | `bool` | image, video, adetail |
+| `circular_y` | `bool` | image, video, adetail |
+| `clip_g_path` | `string` | context |
+| `clip_l_path` | `string` | context |
+| `clip_skip` | `int` | image, video, adetail |
+| `clip_vision_path` | `string` | context |
+| `conditioning_cache_size` | `int` | context |
+| `control_frames` | `tensor` | video |
+| `control_image` | `tensor` | image |
+| `control_net_path` | `string` | context |
+| `control_strength` | `float` | image |
+| `control_type` | `string` | image |
+| `conv_direct` | `bool` | upscaler |
+| `conv_direct_mode` | `string` | context, vid_upscaler |
+| `diffusion_conv_direct` | `bool` | context |
+| `diffusion_flash_attn` | `bool` | context |
+| `diffusion_model_path` | `string` | context, vid_upscaler |
+| `disable_prefetch` | `bool` | context |
+| `disable_segmented_compute` | `bool` | context |
+| `eager_load` | `bool` | context |
+| `embeddings` | `embeddings` | context |
+| `embeddings_connectors_path` | `string` | context |
+| `enable_mmap` | `bool` | context |
+| `end_image` | `tensor` | video |
+| `flash_attn` | `bool` | context |
+| `flash_attn_mode` | `string` | context, vid_upscaler |
+| `force_sdxl_vae_conv_scale` | `bool` | context |
+| `fps` | `int` | video, vid_upscale |
+| `frames` | `tensor` | vid_upscale |
+| `height` | `int` | image, video, adetail |
+| `high_noise_diffusion_model_path` | `string` | context |
+| `high_noise_sample.args` | `string` | video |
+| `high_noise_sample.eta` | `float` | video |
+| `high_noise_sample.guidance.distilled_guidance` | `float` | video |
+| `high_noise_sample.guidance.img_cfg` | `float` | video |
+| `high_noise_sample.guidance.slg.layer_end` | `float` | video |
+| `high_noise_sample.guidance.slg.layer_start` | `float` | video |
+| `high_noise_sample.guidance.slg.layers` | `ints` | video |
+| `high_noise_sample.guidance.slg.scale` | `float` | video |
+| `high_noise_sample.guidance.txt_cfg` | `float` | video |
+| `high_noise_sample.method` | `string` | video |
+| `high_noise_sample.steps` | `int` | video |
+| `hires.denoising_strength` | `float` | image, video |
+| `hires.enabled` | `bool` | image, video |
+| `hires.model_path` | `string` | image, video |
+| `hires.scale` | `float` | image, video |
+| `hires.sigmas` | `floats` | image, video |
+| `hires.steps` | `int` | image, video |
+| `hires.target_height` | `int` | image, video |
+| `hires.target_width` | `int` | image, video |
+| `hires.upscale_tile_size` | `int` | image, video |
+| `hires.upscaler` | `string` | image, video |
+| `image` | `tensor` | upscale, adetail |
+| `image_preprocess.rules` | `string` | image, video |
+| `init_image` | `tensor` | image, video |
+| `ip_adapter_image` | `tensor` | image |
+| `ip_adapter_path` | `string` | context |
+| `ip_adapter_strength` | `float` | image |
+| `linear_scale` | `float` | context |
+| `llm_path` | `string` | context |
+| `llm_vision_path` | `string` | context |
+| `lora_apply_mode` | `string` | context |
+| `loras` | `loras` | image, video, adetail |
+| `mask_image` | `tensor` | image |
+| `max_vram` | `string` | context |
+| `model_args` | `string` | context |
+| `model_path` | `string` | context, upscaler, adetailer |
+| `moe_boundary` | `float` | video |
+| `motion_module_path` | `string` | context |
+| `n_threads` | `int` | context, upscaler, vid_upscaler, adetailer |
+| `negative_prompt` | `string` | image, video, adetail |
+| `params_backend` | `string` | context, upscaler, vid_upscaler, adetailer |
+| `photo_maker_path` | `string` | context |
+| `photomaker.id_embed_path` | `string` | image |
+| `photomaker.id_images` | `tensors` | image |
+| `photomaker.style_strength` | `float` | image |
+| `prediction` | `string` | context |
+| `prompt` | `string` | image, video, adetail |
+| `pulid.id_embedding_path` | `string` | image |
+| `pulid.id_weight` | `float` | image |
+| `pulid_weights_path` | `string` | context |
+| `qwen_image_layers` | `int` | image |
+| `ref_audios` | `audios` | video |
+| `ref_image_args` | `string` | image, video |
+| `ref_images` | `tensors` | image, video |
+| `ref_videos` | `videos` | video |
+| `rng_type` | `string` | context |
+| `rpc_servers` | `string` | context |
+| `sage_attn` | `bool` | context |
+| `sample.args` | `string` | image, video, adetail |
+| `sample.eta` | `float` | image, video, adetail |
+| `sample.flow_shift` | `float` | image, video, adetail |
+| `sample.guidance.distilled_guidance` | `float` | image, video, adetail |
+| `sample.guidance.img_cfg` | `float` | image, video, adetail |
+| `sample.guidance.slg.layer_end` | `float` | image, video, adetail |
+| `sample.guidance.slg.layer_start` | `float` | image, video, adetail |
+| `sample.guidance.slg.layers` | `ints` | image, video, adetail |
+| `sample.guidance.slg.scale` | `float` | image, video, adetail |
+| `sample.guidance.txt_cfg` | `float` | image, video, adetail |
+| `sample.method` | `string` | image, video, adetail |
+| `sample.scheduler` | `string` | image, video, adetail |
+| `sample.shifted_timestep` | `int` | image, video, adetail |
+| `sample.sigmas` | `floats` | image, video, adetail |
+| `sample.steps` | `int` | image, video, adetail |
+| `sampler_rng_type` | `string` | context |
+| `seed` | `int` | image, video, vid_upscale, adetail |
+| `split_mode` | `string` | context |
+| `strength` | `float` | image, video, adetail |
+| `t5xxl_path` | `string` | context |
+| `tae_preview_only` | `bool` | context |
+| `taesd_path` | `string` | context |
+| `temporal_chunk_size` | `int` | vid_upscale |
+| `tensor_type_rules` | `string` | context |
+| `tile_size` | `int` | upscaler |
+| `tokenizer` | `string` | context |
+| `uncond_diffusion_model_path` | `string` | context |
+| `upscale_factor` | `int` | vid_upscaler |
+| `vace_strength` | `float` | video |
+| `vae_conv_direct` | `bool` | context |
+| `vae_format` | `string` | context |
+| `vae_path` | `string` | context, vid_upscaler |
+| `vae_tiling.enabled` | `bool` | image, video, adetail |
+| `vae_tiling.extra_tiling_args` | `string` | image, video, adetail |
+| `vae_tiling.rel_size_h` | `float` | image, video, adetail |
+| `vae_tiling.rel_size_w` | `float` | image, video, adetail |
+| `vae_tiling.target_overlap` | `float` | image, video, adetail |
+| `vae_tiling.temporal_tiling` | `bool` | image, video, adetail |
+| `vae_tiling.tile_size_h` | `int` | image, video, adetail |
+| `vae_tiling.tile_size_w` | `int` | image, video, adetail |
+| `video_frames` | `int` | video |
+| `width` | `int` | image, video, adetail |
+| `window_size` | `int` | vid_upscale |
+| `wtype` | `string` | context |
+
+## Migration
+
+This is a breaking C API change. The former public generation parameter layouts,
+their initialization/formatting helpers, `new_sd_ctx()`, `free_sd_ctx()`,
+`generate_image()`, `generate_video()` and public result containers/destructors
+have been removed. Applications and bindings must migrate and rebuild; no
+compatibility layer is provided.
+
+Create context/image/video options with the corresponding constructor, populate
+them through the shared typed setters, and use `sd_ctx_create()`,
+`sd_generate_image()` and `sd_generate_video()`. Read or retain outputs through
+`sd_result_get_*()` and `sd_tensor_retain()` and release the result with
+`sd_result_free()`. ADetailer and image/video upscalers use purpose-specific opt constructors and
+return the same opaque result type. Their former public parameter structures,
+long positional signatures and bool-returning entry points have been removed.
+Device selection and cancellation controls retain their interfaces. Preview
+registration now uses a mode string and returns a status.
+
+Native generation JSON retains its existing schema and names, including
+`sample_params.sample_method`; translate these to the corresponding option keys
+(`sample.method`, etc.) when replaying metadata. See
+[generation results and metadata](generation_metadata.md).
diff --git a/docs/guides/caching.md b/docs/guides/caching.md
index f69cfbe..66dfc27 100644
--- a/docs/guides/caching.md
+++ b/docs/guides/caching.md
@@ -5,11 +5,11 @@ Caching methods accelerate diffusion inference by reusing intermediate computati
### Conditioning Cache
Conditioning results are cached per model context using an LRU cache. The default
-capacity is **0 (disabled) for `sd-cli`** and **4 entries for `sd-server` and the C
-API**. Set `--conditioning-cache-size N` to change the limit; `0` disables caching.
-For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 4`
-enables the cache in the CLI. The C API option is
-`sd_ctx_params_t::conditioning_cache_size`, initialized by `sd_ctx_params_init()`.
+capacity is **4 entries for `sd-cli`, `sd-server` and the C API**.
+Set `--conditioning-cache-size N` to change the limit; `0` disables caching.
+For example, `sd-cli -m model.safetensors -p "a cat" --conditioning-cache-size 8`
+increases the CLI cache capacity to 8 entries. The C API option is
+`conditioning_cache_size`, set with `sd_opt_set_int()` on context options.
This cache is independent of the diffusion-step `--cache-mode` options below.
### Cache Modes
diff --git a/docs/guides/generation_metadata.md b/docs/guides/generation_metadata.md
index 1f7fc33..f24bda2 100644
--- a/docs/guides/generation_metadata.md
+++ b/docs/guides/generation_metadata.md
@@ -1,30 +1,31 @@
# Generation results and metadata
-`generate_image()`, `generate_video()`, `adetail_image()` and `upscale()` return
-owned results containing the media and its generation record. This is a breaking
-C API change: applications and bindings must update their calls and rebuild.
+`sd_generate_image()`, `sd_generate_video()`, `sd_adetail()`, `sd_upscale()` and
+`sd_upscale_video()` return opaque `sd_result_t` objects containing generated media
+and generation metadata. See the [C API guide](c_api.md) for option creation and
+migration from the removed parameter/result structures.
```c
-sd_img_gen_params_t params;
-sd_img_gen_params_init(¶ms);
-params.prompt = "a small house";
-
-sd_image_generation_result_t* result = NULL;
-if (generate_image(ctx, ¶ms, &result)) {
- for (int i = 0; i < result->count; ++i) {
- const sd_image_result_t* item = &result->items[i];
- /* Consume item->image and item->generation_info together. */
+sd_img_gen_opt_t* opt = sd_img_gen_opt_create();
+sd_result_t* result = NULL;
+if (opt && sd_opt_set_string(opt, "prompt", "a small house") == SD_STATUS_OK &&
+ sd_generate_image(ctx, opt, &result) == SD_STATUS_OK) {
+ for (size_t i = 0; i < sd_result_get_image_count(result); ++i) {
+ const sd_tensor_t* image = sd_result_get_image(result, i);
+ const char* info = sd_result_get_generation_info(result, i);
+ /* Consume image and info together. */
}
- free_sd_image_generation_result(result);
}
+sd_result_free(result);
+sd_opt_free(opt);
```
-`result_out` is required and is set to `NULL` on failure. A successful result
-owns its pixels and UTF-8 JSON strings. The video result additionally owns its
-audio and contains the effective `fps` and `frame_count`. Results remain valid
-after freeing the generation context. Use the matching library result destructor;
-do not separately free fields unless transferring ownership and clearing them.
-Both result destructors accept `NULL`.
+`result_out` is required and set to `NULL` on failure. Successful results own
+their pixels, audio and UTF-8 JSON independently of the generation context.
+Video results expose effective FPS and frame count through accessors; each valid
+frame index returns the shared video generation record. Result views are borrowed
+until `sd_result_free()`. Use `sd_tensor_retain()` to keep media independently,
+and release the retained handle with `sd_tensor_free()`. Free functions accept `NULL`.
## Reproduction parameters
@@ -33,7 +34,7 @@ same loaded context and the same input images, masks, reference media and other
assets. It is created after `GenerationRequest` resolves the request, before
preprocessing or sampling. A failed call returns no result or metadata.
-Fields follow the C generation parameter structs: `prompt`, `negative_prompt`,
+The JSON schema retains its established generation field names: `prompt`, `negative_prompt`,
`width`, `height`, `seed`, `sample_params`, `hires`, `cache`,
`vae_tiling_params`, `image_preprocess`, and the image/video-specific options.
`kind` distinguishes image and video parameters. There is no inferred `task`.
@@ -77,10 +78,10 @@ and shares them across a batch. With a stochastic VAE encoder, replaying a later
image alone using its sampling seed can change those input latents. Per-image
seeds therefore do not guarantee independent reproduction for these paths.
-To replay a record, initialize the matching C parameter struct, populate its
-fields from the JSON, restore the same input buffers, and call generation with
-the same context. Keep custom-sigma/SLG arrays and string storage alive for the
-call. Paths supplied as generation parameters (LoRAs, hires models and identity
+To replay a record, create image/video options, translate JSON fields to the documented C API
+keys, restore the same input buffers, and call generation with the same context.
+Setters copy strings and numeric arrays and retain media tensors; keep options alive
+for the call. Paths supplied as generation parameters (LoRAs, hires models and identity
embeddings) are retained. `models` identifies successfully loaded context model
files by component (`model`, `diffusion_model`, `clip_l`, `vae`, etc.), using
filenames with extensions and omitting directories. Optional components disabled
diff --git a/docs/guides/image_preprocessing.md b/docs/guides/image_preprocessing.md
index c097e5a..dc0aca8 100644
--- a/docs/guides/image_preprocessing.md
+++ b/docs/guides/image_preprocessing.md
@@ -156,23 +156,23 @@ user rules for replay with the same input images; image encodings and channel co
## C API
-Set `image_preprocess` on the existing image/video generation parameters.
-Generation returns media and actual parameters together; see [generation metadata](generation_metadata.md):
+Set the `image_preprocess.rules` key on image/video options. Generation returns
+media and actual parameters together; see [generation metadata](generation_metadata.md):
```c
-sd_img_gen_params_t params;
-sd_img_gen_params_init(¶ms);
+sd_img_gen_opt_t* opt = sd_img_gen_opt_create();
+sd_result_t* result = NULL;
/* Set prompt, original-resolution input images, and generation options. */
-params.image_preprocess.rules = "target=init,mode=crop-resize,filter=lanczos;"
- "target=mask,filter=nearest-exact";
-sd_image_generation_result_t* result = NULL;
-bool ok = generate_image(ctx, ¶ms, &result);
-/* Consume result->items and their generation_info on success. */
-free_sd_image_generation_result(result);
+sd_status_t status = sd_opt_set_string(opt, "image_preprocess.rules",
+ "target=init,mode=crop-resize,filter=lanczos;target=mask,filter=nearest-exact");
+if (status == SD_STATUS_OK)
+ status = sd_generate_image(ctx, opt, &result);
+/* Read outputs through sd_result_get_*() on success. */
+sd_result_free(result);
+sd_opt_free(opt);
```
-Both generation parameter initializers set `image_preprocess.rules` to `NULL`,
-selecting input presets. Rule strings are borrowed for the synchronous call.
+Leaving the key unset selects input presets. Rule strings are copied by setters.
The library owns temporary transformed pixels; caller images and arrays are
not modified. Add `canny=true` to the desired target's rule in
`image_preprocess.rules` to enable Canny.
diff --git a/docs/guides/prompt_images.md b/docs/guides/prompt_images.md
index 726a797..e6c301f 100644
--- a/docs/guides/prompt_images.md
+++ b/docs/guides/prompt_images.md
@@ -47,24 +47,30 @@ smaller source image if this limit is exceeded.
## C API
Register a context-local image resolver with `sd_set_prompt_image_callback()`,
-then call the existing `generate_image()` or `generate_video()` function. The
+then call `sd_generate_image()` or `sd_generate_video()`. The
library passes each marker's `source` string unchanged to the callback, allowing
paths, application resource IDs, or in-memory images. The library does not open
files or fetch URLs itself.
```c
-bool load_prompt_image(const char* source, void* user_data, sd_image_t* image_out);
+bool load_prompt_image(const char* source, void* user_data, const sd_tensor_t** image_out);
-params.prompt = "Draw the subject from <image:subject> in a watercolor landscape.";
-sd_image_generation_result_t* result = NULL;
+sd_img_gen_opt_t* opt = sd_img_gen_opt_create();
+sd_status_t status = sd_opt_set_string(opt, "prompt",
+ "Draw the subject from <image:subject> in a watercolor landscape.");
+sd_result_t* result = NULL;
sd_set_prompt_image_callback(ctx, load_prompt_image, application_images);
-bool ok = generate_image(ctx, ¶ms, &result);
+if (status == SD_STATUS_OK)
+ status = sd_generate_image(ctx, opt, &result);
sd_set_prompt_image_callback(ctx, NULL, NULL);
+sd_result_free(result);
+sd_opt_free(opt);
```
-The loader returns `true` with an image containing interleaved 8-bit pixels and
-one to four channels. The caller owns the pixels. The library copies them before
-calling the loader again, so a loader may reuse its buffer. Repeated sources
+The loader returns `true` with a borrowed U8 tensor of shape
+`[channels,width,height]`, containing interleaved pixels and one to four channels.
+The caller owns the tensor and keeps it valid until the next callback. The
+library copies its pixels before calling the loader again. Repeated sources
within one prompt are loaded once and appear at every specified position.
Resolved image pixels participate in conditioning-cache comparisons.
diff --git a/docs/guides/sage_attention.md b/docs/guides/sage_attention.md
index 2c5d960..2f3a0d0 100644
--- a/docs/guides/sage_attention.md
+++ b/docs/guides/sage_attention.md
@@ -61,7 +61,7 @@ speed, and report model loading and first-step initialization separately.
Quantization, smoothing, and format conversion costs are included in generation
time, so short sequences may not benefit.
-Library callers set `sd_ctx_params_t.sage_attn = true` before `new_sd_ctx()`,
-like `diffusion_flash_attn`. Context creation fails if the requested feature is
-unavailable. Initialize the parameter structure with `sd_ctx_params_init()`.
+Library callers set `sage_attn` with `sd_opt_set_bool(ctx_opt, "sage_attn", true)`
+before `sd_ctx_create()`. Context creation fails if the requested feature is
+unavailable. Create options with `sd_ctx_opt_create()`.
Rebuild library callers against the updated public header.
diff --git a/docs/guides/tokenizers.md b/docs/guides/tokenizers.md
index 272df9d..fb93550 100644
--- a/docs/guides/tokenizers.md
+++ b/docs/guides/tokenizers.md
@@ -52,15 +52,17 @@ sd-cli --diffusion-model sd3.gguf --clip_l clip_l.safetensors \
-p "a cat"
```
-The C API accepts the same string in `sd_ctx_params_t::tokenizer`. A null or
+The C API accepts the same string through the `tokenizer` context-option key. An unset or
empty value keeps an embedded tokenizer where available; PiD and Lens require
a nonempty main tokenizer path. The CLI passes the string through;
`TokenizerConfig` parses and validates it when text encoders are initialized.
```c
-sd_ctx_params_t params;
-sd_ctx_params_init(¶ms);
-params.tokenizer = "clip-l=tokenizer_clip.json,clip-g=tokenizer_clip.json";
+sd_ctx_opt_t* opt = sd_ctx_opt_create();
+sd_status_t status = sd_opt_set_string(opt, "tokenizer",
+ "clip-l=tokenizer_clip.json,clip-g=tokenizer_clip.json");
+/* Set model paths, then call sd_ctx_create(opt, &ctx). */
+sd_opt_free(opt);
```
Rebuild applications against the updated public header when using the updated
diff --git a/docs/models/minimax_h3.md b/docs/models/minimax_h3.md
index dc4c52e..862b04d 100644
--- a/docs/models/minimax_h3.md
+++ b/docs/models/minimax_h3.md
@@ -254,7 +254,7 @@ Reference images keep their aspect ratio and are only downscaled when they
exceed the configured limit; neither mode scales a reference up. The limit is
selected by `ref_image_size`. The default, `match`, uses the pixel area of the
requested generation canvas. Pass `--ref-image-args "ref_image_size=max"` (or
-`ref_image_size` in `sd_vid_gen_params_t.ref_image_args`) to use a 2048 px short
+`ref_image_size` in the `ref_image_args` video-option string) to use a 2048 px short
edge instead, which preserves more identity detail at a higher sampling cost:
reference latents are attended at every sampling step.
@@ -266,7 +266,7 @@ narrower than 32 pixels.
the 32-pixel grid and reports a warning.
The C API exposes the same inputs through `ref_images`, `ref_videos`, and
-`ref_audios` in `sd_vid_gen_params_t`. Each `sd_ref_video_t` supplies its own
+the `ref_audios` video-option key. Each `sd_video_t` supplies its own
frame rate and optional soundtrack; non-24-fps inputs are resampled internally.
#### Sol-H3-VAE decoding
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 27731e1..c667cd4 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -45,7 +45,7 @@ the intended output scale:
The two values can be set independently and apply across model components. The
default `0` preserves each model's built-in settings; `1` explicitly disables the
corresponding scaling. Overrides must be finite positive values. C API users can
-set `linear_scale` and `attn_scale` in `sd_ctx_params_t`.
+set the `linear_scale` and `attn_scale` context-option keys with `sd_opt_set_float()`.
If the problem persists after trying the relevant steps above,
[submit a bug report](https://github.com/leejet/stable-diffusion.cpp/issues/new?template=bug_report.yml).
diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt
index 90164cf..36b9243 100644
--- a/examples/cli/CMakeLists.txt
+++ b/examples/cli/CMakeLists.txt
@@ -14,7 +14,6 @@ if(APPLE)
endif()
target_include_directories(${TARGET} PRIVATE
"${CMAKE_CURRENT_SOURCE_DIR}/.."
- "${PROJECT_SOURCE_DIR}/src"
)
install(TARGETS ${TARGET} RUNTIME)
target_link_libraries(${TARGET} PRIVATE stable-diffusion zip ${CMAKE_THREAD_LIBS_INIT})
diff --git a/examples/cli/README.md b/examples/cli/README.md
index 78dd25a..0316f7b 100644
--- a/examples/cli/README.md
+++ b/examples/cli/README.md
@@ -17,6 +17,13 @@ From the build directory, show the full list of command-line arguments:
See [sampling methods and schedulers](../../docs/guides/sampling.md) for supported
names, defaults, and custom sigma schedules.
+Omitted inference options use the [library defaults](../../docs/guides/c_api.md#keys).
+In particular, an omitted seed requests a random seed; use `--seed 42` for a
+repeatable run. Video generation defaults to 6 requested frames before model
+alignment. SLG layers remain unset unless `--skip-layers` is supplied, and the
+conditioning cache uses the library default. Zero and false are explicit inputs;
+invalid values are reported rather than treated as omitted.
+
Logging defaults to `info`. Use `--log-level <level>` to select `debug`, `verbose`,
`info`, `warn`, or `error` (from most to least detailed). Each level includes
messages at that level and all less detailed levels. `-v` and `--verbose` are
diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp
index 44462eb..a29fd70 100644
--- a/examples/cli/main.cpp
+++ b/examples/cli/main.cpp
@@ -35,28 +35,28 @@ std::regex format_specifier_regex("(?:[^%]|^)(?:%%)*(%\\d{0,3}d)");
struct SDCliParams {
std::string input_frames;
- int video_upscale_factor = 0;
- int vsr_window_size = 16;
- int vsr_temporal_chunk_size = 5;
- SDMode mode = IMG_GEN;
- std::string output_path = "output.png";
- int output_begin_idx = -1;
- int compression_quality = 90;
+ int video_upscale_factor = 0;
+ std::optional<int> vsr_window_size;
+ std::optional<int> vsr_temporal_chunk_size;
+ SDMode mode = IMG_GEN;
+ std::string output_path = "output.png";
+ int output_begin_idx = -1;
+ int compression_quality = 90;
std::string image_path;
std::string metadata_format = "text";
sd_log_level_t log_level = SD_LOG_INFO;
bool convert_name = false;
- preview_t preview_method = PREVIEW_NONE;
- int preview_interval = 1;
- std::string preview_path = "preview.png";
- int preview_fps = 16;
- bool preview_noisy = false;
- bool color = false;
- bool metadata_raw = false;
- bool metadata_brief = false;
- bool metadata_all = false;
+ std::string preview_method = "none";
+ int preview_interval = 1;
+ std::string preview_path = "preview.png";
+ int preview_fps = 16;
+ bool preview_noisy = false;
+ bool color = false;
+ bool metadata_raw = false;
+ bool metadata_brief = false;
+ bool metadata_all = false;
std::string imatrix_out;
std::vector<std::string> imatrix_in;
@@ -167,18 +167,7 @@ struct SDCliParams {
if (++index >= argc) {
return -1;
}
- const char* preview = argv[index];
- int preview_found = -1;
- for (int m = 0; m < PREVIEW_COUNT; m++) {
- if (!strcmp(preview, previews_str[m])) {
- preview_found = m;
- }
- }
- if (preview_found == -1) {
- LOG_ERROR("error: preview method %s", preview);
- return -1;
- }
- preview_method = (preview_t)preview_found;
+ preview_method = argv[index];
return 1;
};
@@ -203,7 +192,7 @@ struct SDCliParams {
on_mode_arg},
{"",
"--preview",
- std::string("preview method. must be one of the following [") + previews_str[0] + ", " + previews_str[1] + ", " + previews_str[2] + ", " + previews_str[3] + "] (default is " + previews_str[PREVIEW_NONE] + ")",
+ std::string("preview method. must be one of the following [") + previews_str[0] + ", " + previews_str[1] + ", " + previews_str[2] + ", " + previews_str[3] + "] (default is " + previews_str[0] + ")",
on_preview_arg},
{"-h",
"--help",
@@ -272,7 +261,7 @@ struct SDCliParams {
<< " log_level: " << log_level_name(log_level) << ",\n"
<< " color: " << (color ? "true" : "false") << ",\n"
<< " convert_name: " << (convert_name ? "true" : "false") << ",\n"
- << " preview_method: " << previews_str[preview_method] << ",\n"
+ << " preview_method: " << preview_method << ",\n"
<< " preview_interval: " << preview_interval << ",\n"
<< " preview_path: \"" << preview_path << "\",\n"
<< " preview_fps: " << preview_fps << ",\n"
@@ -368,10 +357,10 @@ bool load_images_from_dir(const std::string dir,
return false;
}
- images.emplace_back(sd_image_t{(uint32_t)width,
- (uint32_t)height,
- (uint32_t)loaded_channel,
- image_buffer});
+ images.emplace_back(SDImage{(uint32_t)width,
+ (uint32_t)height,
+ (uint32_t)loaded_channel,
+ image_buffer});
if (max_image_num > 0 && static_cast<int>(images.size()) >= max_image_num) {
break;
@@ -402,7 +391,11 @@ std::string format_frame_idx(std::string pattern, int frame_idx) {
int continuous_preview_counter = 0;
-void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy, void* data) {
+void step_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) {
+ std::vector<SDImage> views;
+ for (int i = 0; i < frame_count; ++i)
+ views.push_back(tensor_image_view(tensors[i]));
+ SDImage* image = views.data();
(void)step;
(void)is_noisy;
SDCliParams* cli_params = (SDCliParams*)data;
@@ -424,7 +417,7 @@ void step_callback(int step, int frame_count, sd_image_t* image, bool is_noisy,
LOG_ERROR("save preview image to '%s' failed", path.u8string().c_str());
}
} else {
- int fps = cli_params->preview_method == PREVIEW_PROJ ? cli_params->preview_fps / 4 : cli_params->preview_fps;
+ int fps = cli_params->preview_method == "proj" ? cli_params->preview_fps / 4 : cli_params->preview_fps;
if (create_video_from_sd_images(cli_params->preview_path.c_str(), image, frame_count, fps, cli_params->compression_quality) != 0) {
LOG_ERROR("save preview video to '%s' failed", cli_params->preview_path.c_str());
}
@@ -456,7 +449,7 @@ bool save_results(const SDCliParams& cli_params,
const SDGenerationParams& gen_params,
SDImageVec& results,
int num_results,
- const sd_audio_t* generated_audio,
+ const SDAudio* generated_audio,
const std::string& video_info) {
if (results.empty() || num_results <= 0) {
return false;
@@ -497,7 +490,7 @@ bool save_results(const SDCliParams& cli_params,
}
auto write_image = [&](const fs::path& path, int idx) {
- const sd_image_t& img = results[idx];
+ const SDImage& img = results[idx];
if (!img.data)
return false;
@@ -556,7 +549,7 @@ bool save_results(const SDCliParams& cli_params,
std::string final_ext_lower = ext.u8string();
std::transform(final_ext_lower.begin(), final_ext_lower.end(), final_ext_lower.begin(), ::tolower);
const bool mux_audio = generated_audio != nullptr && (final_ext_lower == ".avi" || final_ext_lower == ".webm");
- if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps, cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
+ if (create_video_from_sd_images(video_path.u8string().c_str(), results.data(), num_results, gen_params.fps.value(), cli_params.compression_quality, mux_audio ? generated_audio : nullptr, params) == 0) {
LOG_INFO("save result video to '%s'", video_path.u8string().c_str());
if (generated_audio != nullptr && !mux_audio) {
fs::path wav_path = video_path;
@@ -591,10 +584,8 @@ bool save_results(const SDCliParams& cli_params,
}
static bool apply_adetailer(sd_ctx_t* sd_ctx,
- const sd_ctx_params_t& sd_ctx_params,
const SDContextParams& ctx_params,
- const SDGenerationParams& gen_params,
- const sd_img_gen_params_t& img_gen_params,
+ SDGenerationParams& gen_params,
SDMode mode,
SDImageVec& results,
int num_results) {
@@ -602,17 +593,12 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx,
return true;
}
- sd_adetailer_params_t ad_params{};
- ad_params.prompt = gen_params.ad_prompt.empty() ? nullptr : gen_params.ad_prompt.c_str();
- ad_params.negative_prompt = gen_params.ad_negative_prompt.empty() ? nullptr : gen_params.ad_negative_prompt.c_str();
- ad_params.extra_ad_args = gen_params.extra_ad_args.c_str();
-
- ADetailerCtxPtr ad_ctx(new_adetailer_ctx(gen_params.ad_model_path.c_str(),
- ctx_params.n_threads,
- sd_ctx_params.backend,
- sd_ctx_params.params_backend));
+ ADetailerCtxPtr ad_ctx(create_sd_adetailer(gen_params.ad_model_path.c_str(),
+ ctx_params.n_threads,
+ ctx_params.effective_backend.c_str(),
+ ctx_params.effective_params_backend.c_str()));
if (ad_ctx == nullptr) {
- LOG_ERROR("new_adetailer_ctx failed");
+ LOG_ERROR("sd_adetailer_create failed");
return false;
}
@@ -620,30 +606,29 @@ static bool apply_adetailer(sd_ctx_t* sd_ctx,
if (results[i].data == nullptr) {
continue;
}
- sd_img_gen_params_t ad_generation_params = img_gen_params;
- ad_generation_params.seed = generation_seed(results.generation_info(i), img_gen_params.seed);
+ SDOptBuilder ad_options(gen_params.to_image_opt(true).release());
+ ad_options.set_int("seed", generation_seed(results.generation_info(i), gen_params.seed.value_or(-1)));
if (mode == IMG_GEN) {
- ad_generation_params.width = 512;
- ad_generation_params.height = 512;
- ad_generation_params.strength = 0.4f;
+ ad_options.unset("width");
+ ad_options.unset("height");
+ ad_options.unset("strength");
}
- sd_image_generation_result_t* detailed = nullptr;
- if (!adetail_image(ad_ctx.get(),
- sd_ctx,
- results[i],
- &ad_params,
- &ad_generation_params,
- &detailed) ||
- detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) {
- free_sd_image_generation_result(detailed);
+ sd_result_t* detailed = nullptr;
+ if (!adetail_sd_image(ad_ctx.get(),
+ sd_ctx,
+ results[i],
+ ad_options.finish(),
+ &detailed) ||
+ detailed == nullptr || sd_result_get_image_count(detailed) <= 0 || sd_tensor_data(sd_result_get_image(detailed, 0)) == nullptr) {
+ sd_result_free(detailed);
LOG_ERROR("ADetailer failed for image %d", i + 1);
return false;
}
- results.append_generation_info(i, detailed->items[0].generation_info, ctx_params.lora_model_dir);
- free(results[i].data);
- results[i] = detailed->items[0].image;
- detailed->items[0].image = {};
- free_sd_image_generation_result(detailed);
+ results.append_generation_info(i, sd_result_get_generation_info(detailed, 0), ctx_params.lora_model_dir);
+ sd_image_free(&results[i]);
+ results[i] = take_result_image(detailed, 0);
+
+ sd_result_free(detailed);
}
return true;
}
@@ -656,7 +641,6 @@ int main(int argc, const char* argv[]) {
SDCliParams cli_params;
SDContextParams ctx_params;
- ctx_params.conditioning_cache_size = 0;
SDGenerationParams gen_params;
sd_set_log_callback(sd_log_cb, (void*)&cli_params);
@@ -685,10 +669,6 @@ int main(int argc, const char* argv[]) {
return 1;
}
ctx_params.prepare_backend_assignments();
- if (ctx_params.diffusion_model_path.empty() || ctx_params.vae_path.empty() || gen_params.fps <= 0) {
- LOG_ERROR("vid_upscale requires --diffusion-model, --vae and positive --fps");
- return 1;
- }
std::vector<SDImageOwner> owned_frames;
if (!gen_params.init_image_path.empty()) {
owned_frames.emplace_back();
@@ -702,44 +682,45 @@ int main(int argc, const char* argv[]) {
}
if (owned_frames.size() > size_t(INT_MAX))
return 1;
- std::vector<sd_image_t> frames;
+ std::vector<SDImage> frames;
for (const auto& frame : owned_frames)
frames.push_back(frame.get());
- sd_video_upscaler_ctx_params_t params;
- sd_video_upscaler_ctx_params_init(¶ms);
+ SDVideoUpscalerParams params;
+
params.diffusion_model_path = ctx_params.diffusion_model_path.c_str();
params.vae_path = ctx_params.vae_path.c_str();
params.upscale_factor = cli_params.video_upscale_factor;
params.n_threads = ctx_params.n_threads;
params.backend = ctx_params.effective_backend.c_str();
params.params_backend = ctx_params.effective_params_backend.c_str();
- params.flash_attn_mode = ctx_params.flash_attn_mode.c_str();
- std::string conv_direct_mode = ctx_params.conv_direct_mode;
+ params.flash_attn_mode = ctx_params.flash_attn_mode ? ctx_params.flash_attn_mode->c_str() : nullptr;
+ std::string conv_direct_mode = ctx_params.conv_direct_mode.value_or("");
if (conv_direct_mode.empty()) {
conv_direct_mode = "auto";
- if (ctx_params.diffusion_conv_direct)
+ if (ctx_params.diffusion_conv_direct.value_or(false))
conv_direct_mode += ",diffusion=on";
- if (ctx_params.vae_conv_direct)
+ if (ctx_params.vae_conv_direct.value_or(false))
conv_direct_mode += ",vae=on";
}
params.conv_direct_mode = conv_direct_mode.c_str();
- std::unique_ptr<video_upscaler_ctx_t, decltype(&free_video_upscaler_ctx)> upscaler(
- new_video_upscaler_ctx(¶ms),
- free_video_upscaler_ctx);
+ std::unique_ptr<sd_vid_upscaler_t, decltype(&sd_vid_upscaler_free)> upscaler(
+ create_sd_vid_upscaler(params),
+ sd_vid_upscaler_free);
if (!upscaler)
return 1;
- sd_video_generation_result_t* output = nullptr;
- sd_video_upscale_params_t upscale_params;
- sd_video_upscale_params_init(&upscale_params);
+ sd_result_t* output = nullptr;
+ SDVideoUpscaleParams upscale_params;
+
upscale_params.frames = frames.data();
upscale_params.frame_count = int(frames.size());
upscale_params.fps = gen_params.fps;
upscale_params.window_size = cli_params.vsr_window_size;
upscale_params.temporal_chunk_size = cli_params.vsr_temporal_chunk_size;
upscale_params.seed = gen_params.seed;
- if (!upscale_video(upscaler.get(), &upscale_params, &output))
+ if (!upscale_sd_video(upscaler.get(), upscale_params, &output))
return 1;
- std::string video_info = output->generation_info;
+ std::string video_info = sd_result_get_generation_info(output, 0);
+ gen_params.fps = sd_result_get_fps(output);
SDImageVec results;
results.adopt_video(output);
return save_results(cli_params, ctx_params, gen_params, results, int(results.size()), nullptr, video_info) ? 0 : 1;
@@ -750,7 +731,7 @@ int main(int argc, const char* argv[]) {
return 1;
}
- if (gen_params.video_frames > 4) {
+ if (cli_params.mode == VID_GEN || gen_params.video_frames > 4) {
size_t last_dot_pos = cli_params.preview_path.find_last_of(".");
std::string base_path = cli_params.preview_path;
std::string file_ext = "";
@@ -763,14 +744,18 @@ int main(int argc, const char* argv[]) {
cli_params.preview_path = base_path + ".avi";
}
}
- cli_params.preview_fps = gen_params.fps;
-
- sd_set_preview_callback(step_callback,
- cli_params.preview_method,
- cli_params.preview_interval,
- !cli_params.preview_noisy,
- cli_params.preview_noisy,
- (void*)&cli_params);
+ if (gen_params.fps)
+ cli_params.preview_fps = *gen_params.fps;
+
+ if (sd_set_preview_callback(step_callback,
+ cli_params.preview_method.c_str(),
+ cli_params.preview_interval,
+ !cli_params.preview_noisy,
+ cli_params.preview_noisy,
+ (void*)&cli_params) != SD_STATUS_OK) {
+ LOG_ERROR("%s", sd_get_last_error());
+ return 1;
+ }
LOG_VERBOSE("version: %s", version_string().c_str());
LOG_VERBOSE("%s", sd_get_system_info());
@@ -813,7 +798,7 @@ int main(int argc, const char* argv[]) {
ctx_params.wtype,
ctx_params.tensor_type_rules.c_str(),
cli_params.convert_name,
- ctx_params.n_threads,
+ ctx_params.n_threads.value_or(0),
loras.empty() ? nullptr : loras.data(),
(int)loras.size());
if (!success) {
@@ -945,7 +930,7 @@ int main(int argc, const char* argv[]) {
gen_params.control_frames.clear();
if (!load_images_from_dir(gen_params.control_video_path,
gen_params.control_frames,
- gen_params.video_frames)) {
+ gen_params.video_frames.value_or(0))) {
return 1;
}
}
@@ -958,60 +943,55 @@ int main(int argc, const char* argv[]) {
}
}
- sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t();
+ SDOptPtr sd_ctx_params = ctx_params.to_context_opt();
SDImageVec results;
- int num_results = 0;
- sd_audio_t* generated_audio = nullptr;
+ int num_results = 0;
+ SDAudio* generated_audio = nullptr;
std::string video_info;
if (cli_params.mode == UPSCALE) {
num_results = 1;
results.push_back(gen_params.init_image.release());
} else {
- SDCtxPtr sd_ctx(new_sd_ctx(&sd_ctx_params));
+ SDCtxPtr sd_ctx(create_sd_context(sd_ctx_params));
if (sd_ctx == nullptr) {
- LOG_INFO("new_sd_ctx_t failed");
+ LOG_INFO("sd_ctx_create failed");
return 1;
}
SDImageOwner prompt_image;
sd_set_prompt_image_callback(
- sd_ctx.get(), [](const char* source, void* data, sd_image_t* image_out) -> bool {
+ sd_ctx.get(), [](const char* source, void* data, const sd_tensor_t** image_out) -> bool {
auto& image = *static_cast<SDImageOwner*>(data);
- sd_image_t loaded{};
+ SDImage loaded{};
if (!load_sd_image_from_file(&loaded, source, 0, 0, 3)) {
return false;
}
image.reset(loaded);
- *image_out = loaded;
+ *image_out = image.tensor();
return true;
},
&prompt_image);
- sd_img_gen_params_t img_gen_params{};
const bool use_img_gen_params = cli_params.mode == IMG_GEN || cli_params.mode == ADETAILER;
- if (use_img_gen_params) {
- img_gen_params = gen_params.to_sd_img_gen_params_t();
- }
-
if (cli_params.mode == IMG_GEN) {
- sd_image_generation_result_t* generated = nullptr;
- generate_image(sd_ctx.get(), &img_gen_params, &generated);
+ sd_result_t* generated = nullptr;
+ generate_sd_image(sd_ctx.get(), gen_params.to_image_opt(), &generated);
results.adopt(generated, ctx_params.lora_model_dir);
num_results = results.count();
} else if (cli_params.mode == ADETAILER) {
num_results = 1;
results.push_back(gen_params.init_image.release());
} else if (cli_params.mode == VID_GEN) {
- sd_vid_gen_params_t vid_gen_params = gen_params.to_sd_vid_gen_params_t();
- sd_video_generation_result_t* generated = nullptr;
- if (generate_video(sd_ctx.get(), &vid_gen_params, &generated)) {
- gen_params.fps = cli_params.preview_fps = generated->fps;
- generated_audio = generated->audio;
- generated->audio = nullptr;
- num_results = generated->frame_count;
+ SDOptPtr vid_gen_params = gen_params.to_video_opt();
+ sd_result_t* generated = nullptr;
+ if (generate_sd_video(sd_ctx.get(), vid_gen_params, &generated)) {
+ gen_params.fps = cli_params.preview_fps = sd_result_get_fps(generated);
+ generated_audio = retain_result_audio(generated);
+
+ num_results = sd_result_get_image_count(generated);
results.adopt_video(generated, ctx_params.lora_model_dir);
video_info = results.generation_info(0);
}
@@ -1024,10 +1004,8 @@ int main(int argc, const char* argv[]) {
if (use_img_gen_params &&
!apply_adetailer(sd_ctx.get(),
- sd_ctx_params,
ctx_params,
gen_params,
- img_gen_params,
cli_params.mode,
results,
num_results)) {
@@ -1035,17 +1013,16 @@ int main(int argc, const char* argv[]) {
}
}
- int upscale_factor = 4; // unused for RealESRGAN_x4plus_anime_6B.pth
if (ctx_params.esrgan_path.size() > 0 && gen_params.upscale_repeats > 0) {
- UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(ctx_params.esrgan_path.c_str(),
- ctx_params.diffusion_conv_direct,
- ctx_params.n_threads,
- gen_params.upscale_tile_size,
- sd_ctx_params.backend,
- sd_ctx_params.params_backend));
+ UpscalerCtxPtr upscaler_ctx(create_sd_upscaler(ctx_params.esrgan_path.c_str(),
+ ctx_params.diffusion_conv_direct,
+ ctx_params.n_threads,
+ gen_params.upscale_tile_size,
+ ctx_params.effective_backend.c_str(),
+ ctx_params.effective_params_backend.c_str()));
if (upscaler_ctx == nullptr) {
- LOG_ERROR("new_upscaler_ctx failed");
+ LOG_ERROR("sd_upscaler_create failed");
} else {
for (int i = 0; i < num_results; i++) {
if (results[i].data == nullptr) {
@@ -1054,20 +1031,17 @@ int main(int argc, const char* argv[]) {
SDImageOwner current_image(results[i]);
results[i] = {0, 0, 0, nullptr};
for (int u = 0; u < gen_params.upscale_repeats; ++u) {
- sd_image_generation_result_t* upscaled = nullptr;
- bool upscale_ok = upscale(upscaler_ctx.get(),
- current_image.get(),
- upscale_factor,
- &upscaled);
- if (!upscale_ok || !upscaled || upscaled->count <= 0 || upscaled->items[0].image.data == nullptr) {
- free_sd_image_generation_result(upscaled);
+ sd_result_t* upscaled = nullptr;
+ bool upscale_ok = upscale_sd_image(upscaler_ctx.get(), current_image.get(), &upscaled);
+ if (!upscale_ok || !upscaled || sd_result_get_image_count(upscaled) <= 0 || sd_tensor_data(sd_result_get_image(upscaled, 0)) == nullptr) {
+ sd_result_free(upscaled);
LOG_ERROR("upscale failed");
break;
}
- results.append_generation_info(i, upscaled->items[0].generation_info);
- sd_image_t upscaled_image = upscaled->items[0].image;
- upscaled->items[0].image = {};
- free_sd_image_generation_result(upscaled);
+ results.append_generation_info(i, sd_result_get_generation_info(upscaled, 0));
+ SDImage upscaled_image = take_result_image(upscaled, 0);
+
+ sd_result_free(upscaled);
current_image.reset(upscaled_image);
}
results[i] = current_image.release(); // Set the final upscaled image as the result
diff --git a/examples/common/common.cpp b/examples/common/common.cpp
index d92c819..0c155c1 100644
--- a/examples/common/common.cpp
+++ b/examples/common/common.cpp
@@ -38,25 +38,6 @@ const char* const modes_str[] = {
"vid_upscale",
};
-static sd_vae_format_t str_to_vae_format(const std::string& value) {
- if (value == "auto") {
- return SD_VAE_FORMAT_AUTO;
- }
- if (value == "flux") {
- return SD_VAE_FORMAT_FLUX;
- }
- if (value == "sd3") {
- return SD_VAE_FORMAT_SD3;
- }
- if (value == "flux2") {
- return SD_VAE_FORMAT_FLUX2;
- }
- if (value == "wan") {
- return SD_VAE_FORMAT_WAN;
- }
- return SD_VAE_FORMAT_COUNT;
-}
-
static void prepend_backend_assignment(std::string& spec, const char* assignment) {
spec = spec.empty() ? assignment : std::string(assignment) + "," + spec;
}
@@ -286,13 +267,13 @@ bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& o
invalid_arg = true;
return;
}
- if (option.concat && !option.target->empty()) {
+ if (option.concat && !option.target.get().empty()) {
if (option.concat > 0 && option.concat <= 0xff) {
- *option.target += static_cast<char>(option.concat);
+ option.target.set(option.target.get() + static_cast<char>(option.concat));
}
- *option.target += argv_to_utf8(i, argv);
+ option.target.set(option.target.get() + argv_to_utf8(i, argv));
} else {
- *option.target = argv_to_utf8(i, argv);
+ option.target.set(argv_to_utf8(i, argv));
}
found_arg = true;
}))
@@ -304,7 +285,7 @@ bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& o
return;
}
try {
- *option.target = std::stoi(argv[i]);
+ option.target.set(std::stoi(argv[i]));
} catch (const std::invalid_argument&) {
invalid_arg = true;
}
@@ -318,7 +299,7 @@ bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& o
return;
}
try {
- *option.target = std::stof(argv[i]);
+ option.target.set(std::stof(argv[i]));
} catch (const std::invalid_argument&) {
invalid_arg = true;
}
@@ -327,8 +308,8 @@ bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& o
break;
if (match_and_apply(options.bool_options, [&](auto& option) {
- *option.target = option.keep_true ? true : false;
- found_arg = true;
+ option.target.set(option.keep_true);
+ found_arg = true;
}))
break;
@@ -360,7 +341,7 @@ bool parse_options(int argc, const char** argv, const std::vector<ArgOptions>& o
return true;
}
-static int parse_scale_override(int argc, const char** argv, int index, float& scale) {
+static int parse_scale_override(int argc, const char** argv, int index, std::optional<float>& scale) {
if (++index >= argc) {
return -1;
}
@@ -379,7 +360,7 @@ static int parse_scale_override(int argc, const char** argv, int index, float& s
return 1;
}
-static int parse_on_off_arg(int argc, const char** argv, int index, const char* option, bool& value) {
+static int parse_on_off_arg(int argc, const char** argv, int index, const char* option, std::optional<bool>& value) {
if (++index >= argc) {
LOG_ERROR("%s requires 'on' or 'off'", option);
return -1;
@@ -587,12 +568,11 @@ ArgOptions SDContextParams::get_options() {
options.int_options = {
{"-t",
"--threads",
- "number of threads to use during computation (default: -1). "
- "If threads <= 0, then threads will be set to the number of CPU physical cores",
+ "positive number of computation threads; unset uses the library CPU core selection",
&n_threads},
{"",
"--conditioning-cache-size",
- "maximum number of conditioning results cached per model context (default: " + std::to_string(conditioning_cache_size) + ", 0 disables caching)",
+ "maximum cached conditioning results; unset uses the library default, 0 disables caching",
&conditioning_cache_size},
};
@@ -667,12 +647,7 @@ ArgOptions SDContextParams::get_options() {
return -1;
}
const char* arg = argv[index];
- rng_type = str_to_rng_type(arg);
- if (rng_type == RNG_TYPE_COUNT) {
- LOG_ERROR("error: invalid rng type %s",
- arg);
- return -1;
- }
+ rng_type = arg;
return 1;
};
@@ -681,12 +656,7 @@ ArgOptions SDContextParams::get_options() {
return -1;
}
const char* arg = argv[index];
- sampler_rng_type = str_to_rng_type(arg);
- if (sampler_rng_type == RNG_TYPE_COUNT) {
- LOG_ERROR("error: invalid sampler rng type %s",
- arg);
- return -1;
- }
+ sampler_rng_type = arg;
return 1;
};
@@ -695,12 +665,7 @@ ArgOptions SDContextParams::get_options() {
return -1;
}
const char* arg = argv[index];
- prediction = str_to_prediction(arg);
- if (prediction == PREDICTION_COUNT) {
- LOG_ERROR("error: invalid prediction type %s",
- arg);
- return -1;
- }
+ prediction = arg;
return 1;
};
@@ -709,12 +674,7 @@ ArgOptions SDContextParams::get_options() {
return -1;
}
const char* arg = argv[index];
- lora_apply_mode = str_to_lora_apply_mode(arg);
- if (lora_apply_mode == LORA_APPLY_MODE_COUNT) {
- LOG_ERROR("error: invalid lora apply model %s",
- arg);
- return -1;
- }
+ lora_apply_mode = arg;
return 1;
};
@@ -835,7 +795,7 @@ ArgOptions SDContextParams::get_options() {
}
bool SDContextParams::print_devices() const {
- if (!sd_add_rpc_servers(rpc_servers.c_str())) {
+ if (!sd_add_rpc_servers(rpc_servers ? rpc_servers->c_str() : nullptr)) {
return false;
}
size_t device_list_size = sd_list_devices(nullptr, 0);
@@ -878,20 +838,12 @@ void SDContextParams::build_embedding_map() {
}
bool SDContextParams::resolve(SDMode mode) {
- if (n_threads <= 0) {
- n_threads = sd_get_num_physical_cores();
- }
-
build_embedding_map();
return true;
}
bool SDContextParams::validate(SDMode mode) {
- if (conditioning_cache_size < 0) {
- LOG_ERROR("error: conditioning-cache-size must be non-negative");
- return false;
- }
if (mode == CONVERT) {
const bool has_convert_input = model_path.length() != 0 ||
clip_l_path.length() != 0 ||
@@ -915,8 +867,8 @@ bool SDContextParams::validate(SDMode mode) {
}
}
- if (str_to_vae_format(vae_format) == SD_VAE_FORMAT_COUNT) {
- LOG_ERROR("error: vae_format must be 'auto', 'flux', 'sd3', 'flux2', or 'wan'");
+ if (!to_context_opt()) {
+ LOG_ERROR("%s", sd_get_last_error());
return false;
}
@@ -934,8 +886,8 @@ bool SDContextParams::resolve_and_validate(SDMode mode) {
}
void SDContextParams::prepare_backend_assignments() {
- effective_backend = backend;
- effective_params_backend = params_backend;
+ effective_backend = backend.value_or("");
+ effective_params_backend = params_backend.value_or("");
if (offload_params_to_cpu) {
prepend_backend_assignment(effective_params_backend, "*=cpu");
@@ -995,39 +947,39 @@ std::string SDContextParams::to_string() const {
<< " lora_model_dir: \"" << lora_model_dir << "\",\n"
<< " hires_upscalers_dir: \"" << hires_upscalers_dir << "\",\n"
<< " photo_maker_path: \"" << photo_maker_path << "\",\n"
- << " rng_type: " << sd_rng_type_name(rng_type) << ",\n"
- << " sampler_rng_type: " << sd_rng_type_name(sampler_rng_type) << ",\n"
+ << " rng_type: " << rng_type << ",\n"
+ << " sampler_rng_type: " << sampler_rng_type << ",\n"
<< " offload_params_to_cpu: " << (offload_params_to_cpu ? "true" : "false") << ",\n"
<< " max_vram: \"" << max_vram << "\",\n"
- << " disable_prefetch: " << (disable_prefetch ? "true" : "false") << ",\n"
- << " disable_segmented_compute: " << (disable_segmented_compute ? "true" : "false") << ",\n"
- << " batched_cfg: " << (batched_cfg ? "true" : "false") << ",\n"
- << " eager_load: " << (eager_load ? "true" : "false") << ",\n"
+ << " disable_prefetch: " << disable_prefetch << ",\n"
+ << " disable_segmented_compute: " << disable_segmented_compute << ",\n"
+ << " batched_cfg: " << batched_cfg << ",\n"
+ << " eager_load: " << eager_load << ",\n"
<< " backend: \"" << backend << "\",\n"
<< " params_backend: \"" << params_backend << "\",\n"
<< " split_mode: \"" << split_mode << "\",\n"
<< " model_args: \"" << model_args << "\",\n"
- << " auto_fit: " << (auto_fit ? "true" : "false") << ",\n"
- << " enable_mmap: " << (enable_mmap ? "true" : "false") << ",\n"
+ << " auto_fit: " << auto_fit << ",\n"
+ << " enable_mmap: " << enable_mmap << ",\n"
<< " control_net_cpu: " << (control_net_cpu ? "true" : "false") << ",\n"
<< " clip_on_cpu: " << (clip_on_cpu ? "true" : "false") << ",\n"
<< " vae_on_cpu: " << (vae_on_cpu ? "true" : "false") << ",\n"
<< " flash_attn_mode: " << flash_attn_mode << ",\n"
- << " conv_direct_mode: " << (conv_direct_mode.empty() ? "auto (legacy flags honored)" : conv_direct_mode) << ",\n"
- << " sage_attn: " << (sage_attn ? "true" : "false") << ",\n"
+ << " conv_direct_mode: " << conv_direct_mode << ",\n"
+ << " sage_attn: " << sage_attn << ",\n"
<< " linear_scale: " << linear_scale << ",\n"
<< " attn_scale: " << attn_scale << ",\n"
- << " diffusion_conv_direct: " << (diffusion_conv_direct ? "true" : "false") << ",\n"
- << " vae_conv_direct: " << (vae_conv_direct ? "true" : "false") << ",\n"
- << " taesd_preview_only: " << (taesd_preview_only ? "true" : "false") << ",\n"
- << " prediction: " << sd_prediction_name(prediction) << ",\n"
- << " lora_apply_mode: " << sd_lora_apply_mode_name(lora_apply_mode) << ",\n"
- << " force_sdxl_vae_conv_scale: " << (force_sdxl_vae_conv_scale ? "true" : "false") << "\n"
+ << " diffusion_conv_direct: " << diffusion_conv_direct << ",\n"
+ << " vae_conv_direct: " << vae_conv_direct << ",\n"
+ << " taesd_preview_only: " << taesd_preview_only << ",\n"
+ << " prediction: " << prediction << ",\n"
+ << " lora_apply_mode: " << lora_apply_mode << ",\n"
+ << " force_sdxl_vae_conv_scale: " << force_sdxl_vae_conv_scale << "\n"
<< "}";
return oss.str();
}
-sd_ctx_params_t SDContextParams::to_sd_ctx_params_t() {
+SDOptPtr SDContextParams::to_context_opt() {
prepare_backend_assignments();
embedding_vec.clear();
embedding_vec.reserve(embedding_map.size());
@@ -1038,67 +990,66 @@ sd_ctx_params_t SDContextParams::to_sd_ctx_params_t() {
embedding_vec.emplace_back(item);
}
- sd_ctx_params_t sd_ctx_params;
- sd_ctx_params_init(&sd_ctx_params);
- sd_ctx_params.model_path = model_path.c_str();
- sd_ctx_params.clip_l_path = clip_l_path.c_str();
- sd_ctx_params.clip_g_path = clip_g_path.c_str();
- sd_ctx_params.clip_vision_path = clip_vision_path.c_str();
- sd_ctx_params.t5xxl_path = t5xxl_path.c_str();
- sd_ctx_params.llm_path = llm_path.c_str();
- sd_ctx_params.llm_vision_path = llm_vision_path.c_str();
- sd_ctx_params.tokenizer = tokenizer.c_str();
- sd_ctx_params.diffusion_model_path = diffusion_model_path.c_str();
- sd_ctx_params.high_noise_diffusion_model_path = high_noise_diffusion_model_path.c_str();
- sd_ctx_params.uncond_diffusion_model_path = uncond_diffusion_model_path.c_str();
- sd_ctx_params.embeddings_connectors_path = embeddings_connectors_path.c_str();
- sd_ctx_params.vae_path = vae_path.c_str();
- sd_ctx_params.audio_vae_path = audio_vae_path.c_str();
- sd_ctx_params.audio_encoder_path = audio_encoder_path.c_str();
- sd_ctx_params.taesd_path = taesd_path.c_str();
- sd_ctx_params.control_net_path = control_net_path.c_str();
- sd_ctx_params.ip_adapter_path = ip_adapter_path.c_str();
- sd_ctx_params.motion_module_path = motion_module_path.c_str();
- sd_ctx_params.embeddings = embedding_vec.data();
- sd_ctx_params.embedding_count = static_cast<uint32_t>(embedding_vec.size());
- sd_ctx_params.photo_maker_path = photo_maker_path.c_str();
- sd_ctx_params.pulid_weights_path = pulid_weights_path.c_str();
- sd_ctx_params.tensor_type_rules = tensor_type_rules.c_str();
- sd_ctx_params.n_threads = n_threads;
- sd_ctx_params.conditioning_cache_size = conditioning_cache_size;
- sd_ctx_params.wtype = wtype;
- sd_ctx_params.rng_type = rng_type;
- sd_ctx_params.sampler_rng_type = sampler_rng_type;
- sd_ctx_params.prediction = prediction;
- sd_ctx_params.lora_apply_mode = lora_apply_mode;
- sd_ctx_params.enable_mmap = enable_mmap;
- sd_ctx_params.flash_attn_mode = flash_attn_mode.c_str();
- sd_ctx_params.conv_direct_mode = conv_direct_mode.c_str();
- sd_ctx_params.sage_attn = sage_attn;
- sd_ctx_params.linear_scale = linear_scale;
- sd_ctx_params.attn_scale = attn_scale;
- sd_ctx_params.tae_preview_only = taesd_preview_only;
- sd_ctx_params.diffusion_conv_direct = diffusion_conv_direct;
- sd_ctx_params.vae_conv_direct = vae_conv_direct;
- sd_ctx_params.force_sdxl_vae_conv_scale = force_sdxl_vae_conv_scale;
- sd_ctx_params.vae_format = str_to_vae_format(vae_format);
- sd_ctx_params.max_vram = max_vram.c_str();
- sd_ctx_params.disable_prefetch = disable_prefetch;
- sd_ctx_params.disable_segmented_compute = disable_segmented_compute;
- sd_ctx_params.batched_cfg = batched_cfg;
- sd_ctx_params.eager_load = eager_load;
- sd_ctx_params.backend = effective_backend.c_str();
- sd_ctx_params.params_backend = effective_params_backend.c_str();
- sd_ctx_params.split_mode = split_mode.c_str();
- sd_ctx_params.auto_fit = auto_fit;
- sd_ctx_params.rpc_servers = rpc_servers.c_str();
- sd_ctx_params.model_args = model_args.empty() ? nullptr : model_args.c_str();
- return sd_ctx_params;
+ SDOptBuilder opt(sd_ctx_opt_create());
+ opt.set_string("model_path", model_path);
+ opt.set_string("clip_l_path", clip_l_path);
+ opt.set_string("clip_g_path", clip_g_path);
+ opt.set_string("clip_vision_path", clip_vision_path);
+ opt.set_string("t5xxl_path", t5xxl_path);
+ opt.set_string("llm_path", llm_path);
+ opt.set_string("llm_vision_path", llm_vision_path);
+ opt.set_string("tokenizer", tokenizer);
+ opt.set_string("diffusion_model_path", diffusion_model_path);
+ opt.set_string("high_noise_diffusion_model_path", high_noise_diffusion_model_path);
+ opt.set_string("uncond_diffusion_model_path", uncond_diffusion_model_path);
+ opt.set_string("embeddings_connectors_path", embeddings_connectors_path);
+ opt.set_string("vae_path", vae_path);
+ opt.set_string("audio_vae_path", audio_vae_path);
+ opt.set_string("audio_encoder_path", audio_encoder_path);
+ opt.set_string("taesd_path", taesd_path);
+ opt.set_string("control_net_path", control_net_path);
+ opt.set_string("ip_adapter_path", ip_adapter_path);
+ opt.set_string("motion_module_path", motion_module_path);
+ opt.set_string("photo_maker_path", photo_maker_path);
+ opt.set_string("pulid_weights_path", pulid_weights_path);
+ opt.set_string("tensor_type_rules", tensor_type_rules);
+ opt.set_int("n_threads", n_threads);
+ opt.set_int("conditioning_cache_size", conditioning_cache_size);
+ if (wtype != SD_TYPE_COUNT)
+ opt.set_string("wtype", sd_type_name(wtype));
+ opt.set_string("rng_type", rng_type);
+ opt.set_string("sampler_rng_type", sampler_rng_type);
+ opt.set_string("prediction", prediction);
+ opt.set_string("lora_apply_mode", lora_apply_mode);
+ opt.set_bool("enable_mmap", enable_mmap);
+ opt.set_string("flash_attn_mode", flash_attn_mode);
+ opt.set_string("conv_direct_mode", conv_direct_mode);
+ opt.set_bool("sage_attn", sage_attn);
+ opt.set_float("linear_scale", linear_scale);
+ opt.set_float("attn_scale", attn_scale);
+ opt.set_bool("tae_preview_only", taesd_preview_only);
+ opt.set_bool("diffusion_conv_direct", diffusion_conv_direct);
+ opt.set_bool("vae_conv_direct", vae_conv_direct);
+ opt.set_bool("force_sdxl_vae_conv_scale", force_sdxl_vae_conv_scale);
+ opt.set_string("vae_format", vae_format);
+ opt.set_string("max_vram", max_vram);
+ opt.set_bool("disable_prefetch", disable_prefetch);
+ opt.set_bool("disable_segmented_compute", disable_segmented_compute);
+ opt.set_bool("batched_cfg", batched_cfg);
+ opt.set_bool("eager_load", eager_load);
+ if (backend || !effective_backend.empty())
+ opt.set_string("backend", effective_backend);
+ if (params_backend || !effective_params_backend.empty())
+ opt.set_string("params_backend", effective_params_backend);
+ opt.set_string("split_mode", split_mode);
+ opt.set_bool("auto_fit", auto_fit);
+ opt.set_string("rpc_servers", rpc_servers);
+ opt.set_string("model_args", model_args);
+ opt.set_embeddings("embeddings", embedding_vec.data(), embedding_vec.size());
+ return opt.finish();
}
SDGenerationParams::SDGenerationParams() {
- sd_sample_params_init(&sample_params);
- sd_sample_params_init(&high_noise_sample_params);
}
ArgOptions SDGenerationParams::get_options() {
@@ -1247,7 +1198,7 @@ ArgOptions SDGenerationParams::get_options() {
&qwen_image_layers},
{"",
"--video-frames",
- "video frames (default: 1)",
+ "video frames; unset uses the library default",
&video_frames},
{"",
"--fps",
@@ -1440,12 +1391,7 @@ ArgOptions SDGenerationParams::get_options() {
return -1;
}
const char* arg = argv[index];
- sample_params.sample_method = str_to_sample_method(arg);
- if (sample_params.sample_method == SAMPLE_METHOD_COUNT) {
- LOG_ERROR("error: invalid sample method %s",
- arg);
- return -1;
- }
+ sample_params.sample_method = arg;
return 1;
};
@@ -1454,12 +1400,7 @@ ArgOptions SDGenerationParams::get_options() {
return -1;
}
const char* arg = argv[index];
- high_noise_sample_params.sample_method = str_to_sample_method(arg);
- if (high_noise_sample_params.sample_method == SAMPLE_METHOD_COUNT) {
- LOG_ERROR("error: invalid high noise sample method %s",
- arg);
- return -1;
- }
+ high_noise_sample_params.sample_method = arg;
return 1;
};
@@ -1468,12 +1409,7 @@ ArgOptions SDGenerationParams::get_options() {
return -1;
}
const char* arg = argv[index];
- sample_params.scheduler = str_to_scheduler(arg);
- if (sample_params.scheduler == SCHEDULER_COUNT) {
- LOG_ERROR("error: invalid scheduler %s",
- arg);
- return -1;
- }
+ sample_params.scheduler = arg;
return 1;
};
@@ -1482,7 +1418,7 @@ ArgOptions SDGenerationParams::get_options() {
return -1;
}
std::string layers_str = argv[index];
- if (layers_str[0] != '[' || layers_str[layers_str.size() - 1] != ']') {
+ if (layers_str.size() < 2 || layers_str.front() != '[' || layers_str.back() != ']') {
return -1;
}
@@ -1495,8 +1431,12 @@ ArgOptions SDGenerationParams::get_options() {
std::vector<int> layers;
for (const auto& token : tokens) {
try {
- layers.push_back(std::stoi(token));
- } catch (const std::invalid_argument&) {
+ size_t used = 0;
+ const int layer = std::stoi(token, &used);
+ if (used != token.size())
+ return -1;
+ layers.push_back(layer);
+ } catch (const std::exception&) {
return -1;
}
}
@@ -1509,7 +1449,7 @@ ArgOptions SDGenerationParams::get_options() {
return -1;
}
std::string layers_str = argv[index];
- if (layers_str[0] != '[' || layers_str[layers_str.size() - 1] != ']') {
+ if (layers_str.size() < 2 || layers_str.front() != '[' || layers_str.back() != ']') {
return -1;
}
@@ -1522,8 +1462,12 @@ ArgOptions SDGenerationParams::get_options() {
std::vector<int> layers;
for (const auto& token : tokens) {
try {
- layers.push_back(std::stoi(token));
- } catch (const std::invalid_argument&) {
+ size_t used = 0;
+ const int layer = std::stoi(token, &used);
+ if (used != token.size())
+ return -1;
+ layers.push_back(layer);
+ } catch (const std::exception&) {
return -1;
}
}
@@ -1531,7 +1475,7 @@ ArgOptions SDGenerationParams::get_options() {
return 1;
};
- auto parse_sigmas_arg = [&](const char* value, std::vector<float>* target, const char* option_name) {
+ auto parse_sigmas_arg = [&](const char* value, std::optional<std::vector<float>>* target, const char* option_name) {
if (target == nullptr || value == nullptr) {
return -1;
}
@@ -1556,7 +1500,7 @@ ArgOptions SDGenerationParams::get_options() {
sigmas_str.pop_back();
}
- size_t before = target->size();
+ std::vector<float> parsed;
std::stringstream ss(sigmas_str);
std::string item;
while (std::getline(ss, item, ',')) {
@@ -1564,7 +1508,7 @@ ArgOptions SDGenerationParams::get_options() {
item.erase(item.find_last_not_of(" \t\n\r\f\v") + 1);
if (!item.empty()) {
try {
- target->push_back(std::stof(item));
+ parsed.push_back(std::stof(item));
} catch (const std::invalid_argument&) {
LOG_ERROR("error: invalid float value '%s' in %s", item.c_str(), option_name);
return -1;
@@ -1575,10 +1519,11 @@ ArgOptions SDGenerationParams::get_options() {
}
}
- if (target->size() == before && !sigmas_str.empty()) {
+ if (parsed.empty() && !sigmas_str.empty()) {
LOG_ERROR("error: could not parse any sigma values from '%s'", value);
return -1;
}
+ *target = std::move(parsed);
return 1;
};
@@ -1763,20 +1708,23 @@ ArgOptions SDGenerationParams::get_options() {
return 1;
};
- std::string sample_methods = sample_method_to_str[0];
- for (int i = 1; i < SAMPLE_METHOD_COUNT; i++) {
- sample_methods += ", " + std::string(sample_method_to_str[i]);
+ std::string sample_methods;
+ for (const char* name : example_options::sample_methods) {
+ if (!sample_methods.empty())
+ sample_methods += ", ";
+ sample_methods += name;
}
-
- std::string schedulers = scheduler_to_str[0];
- for (int i = 1; i < SCHEDULER_COUNT; i++) {
- schedulers += ", " + std::string(scheduler_to_str[i]);
+ std::string schedulers;
+ for (const char* name : example_options::schedulers) {
+ if (!schedulers.empty())
+ schedulers += ", ";
+ schedulers += name;
}
options.manual_options = {
{"-s",
"--seed",
- "RNG seed (default: 42, use random seed for < 0)",
+ "RNG seed; unset or a negative value requests a random seed",
on_seed_arg},
{"",
"--sampling-method",
@@ -1803,11 +1751,11 @@ ArgOptions SDGenerationParams::get_options() {
on_hires_sigmas_arg},
{"",
"--skip-layers",
- "layers to skip for SLG steps (default: [7,8,9])",
+ "layers to skip for SLG steps; unset uses the library default",
on_skip_layers_arg},
{"",
"--high-noise-skip-layers",
- "(high noise) layers to skip for SLG steps (default: [7,8,9])",
+ "(high noise) layers to skip for SLG steps; unset uses the library default",
on_high_noise_skip_layers_arg},
{"-r",
"--ref-image",
@@ -2219,10 +2167,44 @@ bool SDGenerationParams::parse_prompt_images_json(const std::string& json_str) {
return true;
}
+template <typename Field>
+static void load_json_input(const json& object, const char* key, Field& out) {
+ if (!object.contains(key))
+ return;
+ const auto& value = object.at(key);
+ if (value.is_null()) {
+ out = Field{};
+ return;
+ }
+ using T = typename InputType<Field>::type;
+ bool valid = false;
+ if constexpr (std::is_same_v<T, std::string>)
+ valid = value.is_string();
+ else if constexpr (std::is_same_v<T, bool>)
+ valid = value.is_boolean();
+ else if constexpr (std::is_integral_v<T>) {
+ valid = value.is_number_integer();
+ if (valid) {
+ if (value.is_number_unsigned())
+ valid = value.get<uint64_t>() <= static_cast<uint64_t>(std::numeric_limits<T>::max());
+ else {
+ const auto integer = value.get<int64_t>();
+ valid = integer >= std::numeric_limits<T>::min() && integer <= std::numeric_limits<T>::max();
+ }
+ }
+ } else if constexpr (std::is_floating_point_v<T>)
+ valid = value.is_number();
+ else
+ valid = value.is_array();
+ if (!valid)
+ throw std::invalid_argument(std::string(key) + ": invalid value type or range");
+ out = value.get<T>();
+}
+
/** Populates generation params from a JSON payload (server APIs); returns false on any malformed field. */
bool SDGenerationParams::from_json_str(
const std::string& json_str,
- const std::function<std::string(const std::string&)>& lora_path_resolver) {
+ const std::function<std::string(const std::string&)>& lora_path_resolver) try {
json j;
try {
j = json::parse(json_str);
@@ -2236,33 +2218,7 @@ bool SDGenerationParams::from_json_str(
if (j.contains("prompt_images") && !parse_prompt_images_json(j["prompt_images"].dump()))
return false;
- auto load_if_exists = [&](const char* key, auto& out) {
- if (j.contains(key)) {
- using T = std::decay_t<decltype(out)>;
- if constexpr (std::is_same_v<T, std::string>) {
- if (j[key].is_string())
- out = j[key];
- } else if constexpr (std::is_same_v<T, int> || std::is_same_v<T, int64_t>) {
- if (j[key].is_number_integer())
- out = j[key];
- } else if constexpr (std::is_same_v<T, float>) {
- if (j[key].is_number())
- out = j[key];
- } else if constexpr (std::is_same_v<T, bool>) {
- if (j[key].is_boolean())
- out = j[key];
- } else if constexpr (std::is_same_v<T, std::vector<int>>) {
- if (j[key].is_array())
- out = j[key].get<std::vector<int>>();
- } else if constexpr (std::is_same_v<T, std::vector<float>>) {
- if (j[key].is_array())
- out = j[key].get<std::vector<float>>();
- } else if constexpr (std::is_same_v<T, std::vector<std::string>>) {
- if (j[key].is_array())
- out = j[key].get<std::vector<std::string>>();
- }
- }
- };
+ auto load_if_exists = [&](const char* key, auto& out) { load_json_input(j, key, out); };
load_if_exists("prompt", prompt);
load_if_exists("negative_prompt", negative_prompt);
@@ -2297,98 +2253,42 @@ bool SDGenerationParams::from_json_str(
if (j.contains("hires") && j["hires"].is_object()) {
const json& hires_json = j["hires"];
- if (hires_json.contains("enabled") && hires_json["enabled"].is_boolean()) {
- hires_enabled = hires_json["enabled"];
- }
- if (hires_json.contains("upscaler") && hires_json["upscaler"].is_string()) {
- hires_upscaler = hires_json["upscaler"];
- }
- if (hires_json.contains("scale") && hires_json["scale"].is_number()) {
- hires_scale = hires_json["scale"];
- }
- if (hires_json.contains("target_width") && hires_json["target_width"].is_number_integer()) {
- hires_width = hires_json["target_width"];
- }
- if (hires_json.contains("target_height") && hires_json["target_height"].is_number_integer()) {
- hires_height = hires_json["target_height"];
- }
- if (hires_json.contains("steps") && hires_json["steps"].is_number_integer()) {
- hires_steps = hires_json["steps"];
- }
- if (hires_json.contains("denoising_strength") && hires_json["denoising_strength"].is_number()) {
- hires_denoising_strength = hires_json["denoising_strength"];
- }
- if (hires_json.contains("custom_sigmas") && hires_json["custom_sigmas"].is_array()) {
- hires_custom_sigmas = hires_json["custom_sigmas"].get<std::vector<float>>();
- }
- if (hires_json.contains("upscale_tile_size") && hires_json["upscale_tile_size"].is_number_integer()) {
- hires_upscale_tile_size = hires_json["upscale_tile_size"];
- }
+ load_json_input(hires_json, "enabled", hires_enabled);
+ load_json_input(hires_json, "upscaler", hires_upscaler);
+ load_json_input(hires_json, "scale", hires_scale);
+ load_json_input(hires_json, "target_width", hires_width);
+ load_json_input(hires_json, "target_height", hires_height);
+ load_json_input(hires_json, "steps", hires_steps);
+ load_json_input(hires_json, "denoising_strength", hires_denoising_strength);
+ load_json_input(hires_json, "custom_sigmas", hires_custom_sigmas);
+ load_json_input(hires_json, "upscale_tile_size", hires_upscale_tile_size);
}
auto parse_sample_params_json = [&](const json& sample_json,
- sd_sample_params_t& target_params,
+ SDSampleParams& target_params,
std::string& target_extra_sample_args,
- std::vector<int>& target_skip_layers,
- std::vector<float>* target_custom_sigmas) {
- if (sample_json.contains("sample_steps") && sample_json["sample_steps"].is_number_integer()) {
- target_params.sample_steps = sample_json["sample_steps"];
- }
- if (sample_json.contains("eta") && sample_json["eta"].is_number()) {
- target_params.eta = sample_json["eta"];
- }
- if (sample_json.contains("shifted_timestep") && sample_json["shifted_timestep"].is_number_integer()) {
- target_params.shifted_timestep = sample_json["shifted_timestep"];
- }
- if (sample_json.contains("flow_shift") && sample_json["flow_shift"].is_number()) {
- target_params.flow_shift = sample_json["flow_shift"];
- }
- if (sample_json.contains("extra_sample_args") && sample_json["extra_sample_args"].is_string()) {
- target_extra_sample_args = sample_json["extra_sample_args"].get<std::string>();
- }
- if (target_custom_sigmas != nullptr &&
- sample_json.contains("custom_sigmas") &&
- sample_json["custom_sigmas"].is_array() &&
- !sample_json["custom_sigmas"].empty()) { // empty [] = "not specified", keep server/CLI default
- *target_custom_sigmas = sample_json["custom_sigmas"].get<std::vector<float>>();
- }
- if (sample_json.contains("sample_method") && sample_json["sample_method"].is_string()) {
- enum sample_method_t tmp = str_to_sample_method(sample_json["sample_method"].get<std::string>().c_str());
- if (tmp != SAMPLE_METHOD_COUNT) {
- target_params.sample_method = tmp;
- }
- }
- if (sample_json.contains("scheduler") && sample_json["scheduler"].is_string()) {
- enum scheduler_t tmp = str_to_scheduler(sample_json["scheduler"].get<std::string>().c_str());
- if (tmp != SCHEDULER_COUNT) {
- target_params.scheduler = tmp;
- }
- }
+ std::optional<std::vector<int>>& target_skip_layers,
+ std::optional<std::vector<float>>* target_custom_sigmas) {
+ load_json_input(sample_json, "sample_steps", target_params.sample_steps);
+ load_json_input(sample_json, "eta", target_params.eta);
+ load_json_input(sample_json, "shifted_timestep", target_params.shifted_timestep);
+ load_json_input(sample_json, "flow_shift", target_params.flow_shift);
+ load_json_input(sample_json, "extra_sample_args", target_extra_sample_args);
+ if (target_custom_sigmas)
+ load_json_input(sample_json, "custom_sigmas", *target_custom_sigmas);
+ load_json_input(sample_json, "sample_method", target_params.sample_method);
+ load_json_input(sample_json, "scheduler", target_params.scheduler);
if (sample_json.contains("guidance") && sample_json["guidance"].is_object()) {
const json& guidance_json = sample_json["guidance"];
- if (guidance_json.contains("txt_cfg") && guidance_json["txt_cfg"].is_number()) {
- target_params.guidance.txt_cfg = guidance_json["txt_cfg"];
- }
- if (guidance_json.contains("img_cfg") && guidance_json["img_cfg"].is_number()) {
- target_params.guidance.img_cfg = guidance_json["img_cfg"];
- }
- if (guidance_json.contains("distilled_guidance") && guidance_json["distilled_guidance"].is_number()) {
- target_params.guidance.distilled_guidance = guidance_json["distilled_guidance"];
- }
+ load_json_input(guidance_json, "txt_cfg", target_params.guidance.txt_cfg);
+ load_json_input(guidance_json, "img_cfg", target_params.guidance.img_cfg);
+ load_json_input(guidance_json, "distilled_guidance", target_params.guidance.distilled_guidance);
if (guidance_json.contains("slg") && guidance_json["slg"].is_object()) {
const json& slg_json = guidance_json["slg"];
- if (slg_json.contains("layers") && slg_json["layers"].is_array()) {
- target_skip_layers = slg_json["layers"].get<std::vector<int>>();
- }
- if (slg_json.contains("layer_start") && slg_json["layer_start"].is_number()) {
- target_params.guidance.slg.layer_start = slg_json["layer_start"];
- }
- if (slg_json.contains("layer_end") && slg_json["layer_end"].is_number()) {
- target_params.guidance.slg.layer_end = slg_json["layer_end"];
- }
- if (slg_json.contains("scale") && slg_json["scale"].is_number()) {
- target_params.guidance.slg.scale = slg_json["scale"];
- }
+ load_json_input(slg_json, "layers", target_skip_layers);
+ load_json_input(slg_json, "layer_start", target_params.guidance.slg.layer_start);
+ load_json_input(slg_json, "layer_end", target_params.guidance.slg.layer_end);
+ load_json_input(slg_json, "scale", target_params.guidance.slg.scale);
}
}
};
@@ -2406,30 +2306,14 @@ bool SDGenerationParams::from_json_str(
if (j.contains("vae_tiling_params") && j["vae_tiling_params"].is_object()) {
const json& tiling_json = j["vae_tiling_params"];
- if (tiling_json.contains("enabled") && tiling_json["enabled"].is_boolean()) {
- vae_tiling_params.enabled = tiling_json["enabled"];
- }
- if (tiling_json.contains("temporal_tiling") && tiling_json["temporal_tiling"].is_boolean()) {
- vae_tiling_params.temporal_tiling = tiling_json["temporal_tiling"];
- }
- if (tiling_json.contains("tile_size_w") && tiling_json["tile_size_w"].is_number_integer()) {
- vae_tiling_params.tile_size_w = tiling_json["tile_size_w"];
- }
- if (tiling_json.contains("tile_size_h") && tiling_json["tile_size_h"].is_number_integer()) {
- vae_tiling_params.tile_size_h = tiling_json["tile_size_h"];
- }
- if (tiling_json.contains("target_overlap") && tiling_json["target_overlap"].is_number()) {
- vae_tiling_params.target_overlap = tiling_json["target_overlap"];
- }
- if (tiling_json.contains("rel_size_w") && tiling_json["rel_size_w"].is_number()) {
- vae_tiling_params.rel_size_w = tiling_json["rel_size_w"];
- }
- if (tiling_json.contains("rel_size_h") && tiling_json["rel_size_h"].is_number()) {
- vae_tiling_params.rel_size_h = tiling_json["rel_size_h"];
- }
- if (tiling_json.contains("extra_tiling_args") && tiling_json["extra_tiling_args"].is_string()) {
- extra_tiling_args = tiling_json["extra_tiling_args"].get<std::string>();
- }
+ load_json_input(tiling_json, "enabled", vae_tiling_params.enabled);
+ load_json_input(tiling_json, "temporal_tiling", vae_tiling_params.temporal_tiling);
+ load_json_input(tiling_json, "tile_size_w", vae_tiling_params.tile_size_w);
+ load_json_input(tiling_json, "tile_size_h", vae_tiling_params.tile_size_h);
+ load_json_input(tiling_json, "target_overlap", vae_tiling_params.target_overlap);
+ load_json_input(tiling_json, "rel_size_w", vae_tiling_params.rel_size_w);
+ load_json_input(tiling_json, "rel_size_h", vae_tiling_params.rel_size_h);
+ load_json_input(tiling_json, "extra_tiling_args", extra_tiling_args);
}
if (!parse_lora_json_field(j, lora_path_resolver, lora_map, high_noise_lora_map)) {
@@ -2460,6 +2344,9 @@ bool SDGenerationParams::from_json_str(
}
return true;
+} catch (const std::exception& error) {
+ LOG_ERROR("invalid generation parameters: %s", error.what());
+ return false;
}
bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_dir) {
@@ -2543,27 +2430,26 @@ bool SDGenerationParams::extract_and_remove_lora(const std::string& lora_model_d
}
bool SDGenerationParams::width_and_height_are_set() const {
- return width > 0 && height > 0;
+ return width.has_value() && height.has_value();
}
void SDGenerationParams::set_width_and_height_if_unset(int w, int h) {
- if (!width_and_height_are_set()) {
- LOG_INFO("set width x height to %d x %d", w, h);
- width = w;
+ if (!width)
+ width = w;
+ if (!height)
height = h;
- }
}
-int SDGenerationParams::get_resolved_width() const {
- return (width > 0) ? width : 512;
+int SDGenerationParams::get_requested_width() const {
+ return width.value_or(0);
}
-int SDGenerationParams::get_resolved_height() const {
- return (height > 0) ? height : 512;
+int SDGenerationParams::get_requested_height() const {
+ return height.value_or(0);
}
bool SDGenerationParams::initialize_cache_params() {
- sd_cache_params_init(&cache_params);
+ cache_params = SDCacheParams{};
auto parse_named_params = [&](const std::string& opt_str) -> bool {
std::stringstream ss(opt_str);
@@ -2629,19 +2515,19 @@ bool SDGenerationParams::initialize_cache_params() {
if (!cache_mode.empty()) {
if (cache_mode == "disabled") {
- cache_params.mode = SD_CACHE_DISABLED;
+ cache_params.mode = "disabled";
} else if (cache_mode == "easycache") {
- cache_params.mode = SD_CACHE_EASYCACHE;
+ cache_params.mode = "easycache";
} else if (cache_mode == "ucache") {
- cache_params.mode = SD_CACHE_UCACHE;
+ cache_params.mode = "ucache";
} else if (cache_mode == "dbcache") {
- cache_params.mode = SD_CACHE_DBCACHE;
+ cache_params.mode = "dbcache";
} else if (cache_mode == "taylorseer") {
- cache_params.mode = SD_CACHE_TAYLORSEER;
+ cache_params.mode = "taylorseer";
} else if (cache_mode == "cache-dit") {
- cache_params.mode = SD_CACHE_CACHE_DIT;
+ cache_params.mode = "cache_dit";
} else if (cache_mode == "spectrum") {
- cache_params.mode = SD_CACHE_SPECTRUM;
+ cache_params.mode = "spectrum";
} else {
LOG_ERROR("error: invalid cache mode '%s'", cache_mode.c_str());
return false;
@@ -2652,9 +2538,9 @@ bool SDGenerationParams::initialize_cache_params() {
return false;
}
- if (cache_params.mode == SD_CACHE_DBCACHE ||
- cache_params.mode == SD_CACHE_TAYLORSEER ||
- cache_params.mode == SD_CACHE_CACHE_DIT) {
+ if (cache_params.mode == "dbcache" ||
+ cache_params.mode == "taylorseer" ||
+ cache_params.mode == "cache_dit") {
cache_params.scm_policy_dynamic = scm_policy_dynamic;
}
@@ -2664,47 +2550,35 @@ bool SDGenerationParams::initialize_cache_params() {
bool SDGenerationParams::resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict, bool validate_missing_loras) {
vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
- if (high_noise_sample_params.sample_steps <= 0) {
- high_noise_sample_params.sample_steps = -1;
- }
-
if (!initialize_cache_params()) {
return false;
}
- if (strict) {
- batch_count = std::clamp(batch_count, 1, 8);
- sample_params.sample_steps = std::clamp(sample_params.sample_steps, 1, 100);
- }
-
- resolved_control_type = SD_CONTROL_TYPE_NONE;
- if (!control_type.empty()) {
- resolved_control_type = str_to_sd_control_type(control_type.c_str());
- if (resolved_control_type == SD_CONTROL_TYPE_COUNT) {
- LOG_ERROR("invalid control type '%s' (openpose, depth, scribble, canny, normal, segment, tile, repaint)", control_type.c_str());
- return false;
- }
+ if (strict && ((batch_count && (*batch_count < 1 || *batch_count > 8)) ||
+ (sample_params.sample_steps && (*sample_params.sample_steps < 1 || *sample_params.sample_steps > 100)))) {
+ LOG_ERROR("server requests require batch_count in [1, 8] and sample_steps in [1, 100]");
+ return false;
}
hires_upscaler_model_path.clear();
- if (hires_enabled) {
- if (hires_upscaler.empty()) {
- hires_upscaler = "Latent";
- }
- resolved_hires_upscaler = str_to_sd_hires_upscaler(hires_upscaler.c_str());
- if (resolved_hires_upscaler == SD_HIRES_UPSCALER_NONE) {
- hires_enabled = false;
- } else if (resolved_hires_upscaler == SD_HIRES_UPSCALER_COUNT) {
+ resolved_hires_upscaler.clear();
+ if (hires_upscaler) {
+ static const std::map<std::string, std::string> names = {
+ {"None", "none"}, {"Latent", "latent"}, {"Latent (nearest)", "latent_nearest"}, {"Latent (nearest-exact)", "latent_nearest_exact"}, {"Latent (antialiased)", "latent_antialiased"}, {"Latent (bicubic)", "latent_bicubic"}, {"Latent (bicubic antialiased)", "latent_bicubic_antialiased"}, {"Lanczos", "lanczos"}, {"Nearest", "nearest"}, {"Model", "model"}};
+ const auto it = names.find(*hires_upscaler);
+ if (it != names.end()) {
+ resolved_hires_upscaler = it->second;
+ } else if (std::any_of(names.begin(), names.end(), [&](const auto& name) { return name.second == *hires_upscaler; })) {
+ resolved_hires_upscaler = *hires_upscaler;
+ } else {
static const std::vector<std::string> valid_ext = {".gguf", ".safetensors", ".pt", ".pth"};
- if (!resolve_model_file_from_dir(hires_upscaler,
- hires_upscalers_dir,
- valid_ext,
- "hires upscaler",
- hires_upscaler_model_path)) {
+ if (hires_enabled.value_or(false) &&
+ !resolve_model_file_from_dir(*hires_upscaler, hires_upscalers_dir, valid_ext, "hires upscaler", hires_upscaler_model_path))
return false;
- }
- resolved_hires_upscaler = SD_HIRES_UPSCALER_MODEL;
+ resolved_hires_upscaler = "model";
}
+ if (resolved_hires_upscaler == "none")
+ hires_enabled = false;
}
prompt_with_lora = prompt;
@@ -2722,54 +2596,6 @@ bool SDGenerationParams::validate(SDMode mode) {
LOG_ERROR("--image-preprocess requires img_gen or vid_gen mode");
return false;
}
- if (batch_count <= 0) {
- LOG_ERROR("error: batch_count must be greater than 0");
- return false;
- }
-
- if (qwen_image_layers < 0) {
- LOG_ERROR("error: qwen_image_layers must be non-negative");
- return false;
- }
-
- if (sample_params.sample_steps <= 0) {
- LOG_ERROR("error: the sample_steps must be greater than 0\n");
- return false;
- }
-
- if (strength < 0.f || strength > 1.f) {
- LOG_ERROR("error: can only work with strength in [0.0, 1.0]\n");
- return false;
- }
-
- if (sample_params.guidance.txt_cfg < 0.f) {
- LOG_ERROR("error: cfg_scale must be positive");
- return false;
- }
-
- if (!cache_mode.empty()) {
- if (cache_mode == "easycache" || cache_mode == "ucache") {
- if (cache_params.reuse_threshold < 0.0f) {
- LOG_ERROR("error: cache threshold must be non-negative");
- return false;
- }
- if (cache_params.start_percent < 0.0f || cache_params.start_percent >= 1.0f ||
- cache_params.end_percent <= 0.0f || cache_params.end_percent > 1.0f ||
- cache_params.start_percent >= cache_params.end_percent) {
- LOG_ERROR("error: cache start/end percents must satisfy 0.0 <= start < end <= 1.0");
- return false;
- }
- }
- }
-
- if (mode == VID_GEN && video_frames <= 0) {
- return false;
- }
-
- if (mode == VID_GEN && fps <= 0) {
- return false;
- }
-
if (ref_video_audio_paths.size() + ref_video_audios.size() > ref_video_paths.size() + ref_videos.size()) {
LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video");
return false;
@@ -2781,46 +2607,10 @@ bool SDGenerationParams::validate(SDMode mode) {
return false;
}
- if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) {
- LOG_ERROR("error: shifted_timestep must be in range [0, 1000]");
- return false;
- }
-
if (upscale_repeats < 1) {
return false;
}
- if (upscale_tile_size < 1) {
- return false;
- }
-
- if (hires_enabled) {
- if (hires_width < 0 || hires_height < 0) {
- LOG_ERROR("error: hires target width and height must be >= 0");
- return false;
- }
- if (hires_scale <= 0.f && hires_width <= 0 && hires_height <= 0) {
- LOG_ERROR("error: hires scale must be positive when target size is not set");
- return false;
- }
- if (hires_steps < 0) {
- LOG_ERROR("error: hires steps must be >= 0");
- return false;
- }
- if (hires_denoising_strength <= 0.f || hires_denoising_strength > 1.f) {
- LOG_ERROR("error: hires denoising strength must be in (0.0, 1.0]");
- return false;
- }
- if (!hires_custom_sigmas.empty() && hires_custom_sigmas.size() < 2) {
- LOG_ERROR("error: hires custom sigmas must contain at least two values");
- return false;
- }
- if (hires_upscale_tile_size < 1) {
- LOG_ERROR("error: hires upscale tile size must be positive");
- return false;
- }
- }
-
if (mode == UPSCALE || mode == ADETAILER) {
if (init_image_path.length() == 0) {
LOG_ERROR("error: %s mode needs an init image (--init-img)\n",
@@ -2834,6 +2624,13 @@ bool SDGenerationParams::validate(SDMode mode) {
return false;
}
+ if (mode == IMG_GEN || mode == ADETAILER || mode == VID_GEN) {
+ auto opt = mode == VID_GEN ? to_video_opt() : to_image_opt(mode == ADETAILER);
+ if (!opt) {
+ LOG_ERROR("%s", sd_get_last_error());
+ return false;
+ }
+ }
return true;
}
@@ -2850,10 +2647,68 @@ bool SDGenerationParams::resolve_and_validate(SDMode mode,
return true;
}
-sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() {
- sd_img_gen_params_t params;
- sd_img_gen_params_init(¶ms);
+static void set_sample_options(SDOptBuilder& opt, const std::string& prefix, const SDSampleParams& params, const std::optional<std::vector<int>>& layers, const std::optional<std::vector<float>>& sigmas, const std::string& extra_args) {
+ if (!extra_args.empty())
+ opt.set_string((prefix + "args").c_str(), extra_args);
+ opt.set_float((prefix + "eta").c_str(), params.eta);
+ opt.set_float((prefix + "guidance.distilled_guidance").c_str(), params.guidance.distilled_guidance);
+ opt.set_float((prefix + "guidance.img_cfg").c_str(), params.guidance.img_cfg);
+ opt.set_float((prefix + "guidance.slg.layer_end").c_str(), params.guidance.slg.layer_end);
+ opt.set_float((prefix + "guidance.slg.layer_start").c_str(), params.guidance.slg.layer_start);
+ if (layers)
+ opt.set_ints((prefix + "guidance.slg.layers").c_str(), layers->data(), layers->size());
+ opt.set_float((prefix + "guidance.slg.scale").c_str(), params.guidance.slg.scale);
+ opt.set_float((prefix + "guidance.txt_cfg").c_str(), params.guidance.txt_cfg);
+ opt.set_string((prefix + "method").c_str(), params.sample_method);
+ if (prefix == "sample.") {
+ opt.set_string("sample.scheduler", params.scheduler);
+ opt.set_float("sample.flow_shift", params.flow_shift);
+ opt.set_int("sample.shifted_timestep", params.shifted_timestep);
+ if (sigmas)
+ opt.set_floats((prefix + "sigmas").c_str(), sigmas->data(), sigmas->size());
+ }
+ opt.set_int((prefix + "steps").c_str(), params.sample_steps);
+}
+
+static void set_cache_options(SDOptBuilder& opt, const SDCacheParams& params) {
+ opt.set_int("cache.bn_compute_blocks", params.Bn_compute_blocks);
+ opt.set_float("cache.end_percent", params.end_percent);
+ opt.set_float("cache.error_decay_rate", params.error_decay_rate);
+ opt.set_int("cache.fn_compute_blocks", params.Fn_compute_blocks);
+ opt.set_int("cache.max_cached_steps", params.max_cached_steps);
+ opt.set_int("cache.max_continuous_cached_steps", params.max_continuous_cached_steps);
+ opt.set_int("cache.max_warmup_steps", params.max_warmup_steps);
+ opt.set_string("cache.mode", params.mode);
+ opt.set_bool("cache.reset_error_on_compute", params.reset_error_on_compute);
+ opt.set_float("cache.residual_diff_threshold", params.residual_diff_threshold);
+ opt.set_float("cache.reuse_threshold", params.reuse_threshold);
+ opt.set_string("cache.scm_mask", params.scm_mask);
+ opt.set_bool("cache.scm_policy_dynamic", params.scm_policy_dynamic);
+ opt.set_float("cache.spectrum_flex_window", params.spectrum_flex_window);
+ opt.set_float("cache.spectrum_lam", params.spectrum_lam);
+ opt.set_int("cache.spectrum_m", params.spectrum_m);
+ opt.set_float("cache.spectrum_stop_percent", params.spectrum_stop_percent);
+ opt.set_float("cache.spectrum_w", params.spectrum_w);
+ opt.set_int("cache.spectrum_warmup_steps", params.spectrum_warmup_steps);
+ opt.set_int("cache.spectrum_window_size", params.spectrum_window_size);
+ opt.set_float("cache.start_percent", params.start_percent);
+ opt.set_int("cache.taylorseer_n_derivatives", params.taylorseer_n_derivatives);
+ opt.set_int("cache.taylorseer_skip_interval", params.taylorseer_skip_interval);
+ opt.set_bool("cache.use_relative_threshold", params.use_relative_threshold);
+}
+
+static void set_tiling_options(SDOptBuilder& opt, const SDTilingParams& params) {
+ opt.set_bool("vae_tiling.enabled", params.enabled);
+ opt.set_string("vae_tiling.extra_tiling_args", params.extra_tiling_args);
+ opt.set_float("vae_tiling.rel_size_h", params.rel_size_h);
+ opt.set_float("vae_tiling.rel_size_w", params.rel_size_w);
+ opt.set_float("vae_tiling.target_overlap", params.target_overlap);
+ opt.set_bool("vae_tiling.temporal_tiling", params.temporal_tiling);
+ opt.set_int("vae_tiling.tile_size_h", params.tile_size_h);
+ opt.set_int("vae_tiling.tile_size_w", params.tile_size_w);
+}
+SDOptPtr SDGenerationParams::to_image_opt(bool adetail) {
lora_vec.clear();
lora_vec.reserve(lora_map.size() + high_noise_lora_map.size());
for (const auto& kv : lora_map) {
@@ -2875,85 +2730,101 @@ sd_img_gen_params_t SDGenerationParams::to_sd_img_gen_params_t() {
pm_id_image_views.push_back(image.get());
}
- sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data();
- sample_params.guidance.slg.layer_count = skip_layers.size();
- high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data();
- high_noise_sample_params.guidance.slg.layer_count = high_noise_skip_layers.size();
- sample_params.custom_sigmas = custom_sigmas.empty() ? nullptr : custom_sigmas.data();
- sample_params.custom_sigmas_count = static_cast<int>(custom_sigmas.size());
- sample_params.extra_sample_args = extra_sample_args.empty() ? nullptr : extra_sample_args.c_str();
- high_noise_sample_params.extra_sample_args = high_noise_extra_sample_args.empty() ? nullptr : high_noise_extra_sample_args.c_str();
- vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
- cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str();
-
- sd_pm_params_t pm_params = {
- pm_id_image_views.empty() ? nullptr : pm_id_image_views.data(),
- static_cast<int>(pm_id_image_views.size()),
- pm_id_embed_path.empty() ? nullptr : pm_id_embed_path.c_str(),
- pm_style_strength,
- };
-
- sd_pulid_params_t pulid_params = {
- pulid_id_embedding_path.empty() ? nullptr : pulid_id_embedding_path.c_str(),
- pulid_id_weight,
- };
-
- if (increase_ref_index) {
- if (!ref_image_args.empty()) {
- ref_image_args += ",";
- }
- ref_image_args += "ref_index_mode=increase";
- LOG_WARN("Notice: --increase-ref-index is deprecated. Use --ref-image-args \"ref_index_mode=increase\" instead.");
- }
-
- params.loras = lora_vec.empty() ? nullptr : lora_vec.data();
- params.lora_count = static_cast<uint32_t>(lora_vec.size());
- params.prompt = prompt.c_str();
- params.negative_prompt = negative_prompt.c_str();
- params.clip_skip = clip_skip;
- params.init_image = init_image.get();
- params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data();
- params.ref_images_count = static_cast<int>(ref_image_views.size());
- params.ref_image_args = ref_image_args.c_str();
- params.mask_image = mask_image.get();
- params.width = get_resolved_width();
- params.height = get_resolved_height();
- params.sample_params = sample_params;
- params.strength = strength;
- params.seed = seed;
- params.batch_count = batch_count;
- params.qwen_image_layers = qwen_image_layers;
- params.control_image = control_image.get();
- params.control_strength = control_strength;
- params.control_type = resolved_control_type;
- params.ip_adapter_image = ip_adapter_image.get();
- params.ip_adapter_strength = ip_adapter_strength;
- params.pm_params = pm_params;
- params.pulid_params = pulid_params;
- params.vae_tiling_params = vae_tiling_params;
- params.cache = cache_params;
-
- params.hires.enabled = hires_enabled;
- params.hires.upscaler = resolved_hires_upscaler;
- params.hires.model_path = hires_upscaler_model_path.empty() ? nullptr : hires_upscaler_model_path.c_str();
- params.hires.scale = hires_scale;
- params.hires.target_width = hires_width;
- params.hires.target_height = hires_height;
- params.hires.steps = hires_steps;
- params.hires.denoising_strength = hires_denoising_strength;
- params.hires.upscale_tile_size = hires_upscale_tile_size;
- params.hires.custom_sigmas = hires_custom_sigmas.empty() ? nullptr : hires_custom_sigmas.data();
- params.hires.custom_sigmas_count = static_cast<int>(hires_custom_sigmas.size());
- params.circular_x = circular || circular_x;
- params.circular_y = circular || circular_y;
- params.image_preprocess = {image_preprocess.c_str()};
- return params;
+ vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
+ cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str();
+
+ std::string effective_ref_image_args = ref_image_args;
+ if (!adetail && increase_ref_index) {
+ if (!effective_ref_image_args.empty())
+ effective_ref_image_args += ",";
+ effective_ref_image_args += "ref_index_mode=increase";
+ }
+
+ SDOptBuilder opt(adetail ? sd_adetail_opt_create() : sd_img_gen_opt_create());
+ opt.set_string("prompt", prompt);
+ opt.set_string("negative_prompt", negative_prompt);
+ opt.set_int("clip_skip", clip_skip);
+ if (!adetail)
+ opt.set_image("init_image", init_image.get());
+ if (!adetail)
+ opt.set_string("ref_image_args", effective_ref_image_args);
+ if (!adetail)
+ opt.set_image("mask_image", mask_image.get());
+ opt.set_int("width", width);
+ opt.set_int("height", height);
+ opt.set_float("strength", strength);
+ opt.set_int("seed", seed);
+ if (!adetail)
+ opt.set_int("batch_count", batch_count);
+ if (!adetail)
+ opt.set_int("qwen_image_layers", qwen_image_layers);
+ if (!adetail)
+ opt.set_image("control_image", control_image.get());
+ if (!adetail)
+ opt.set_float("control_strength", control_strength);
+ if (!adetail)
+ opt.set_string("control_type", control_type);
+ if (!adetail)
+ opt.set_image("ip_adapter_image", ip_adapter_image.get());
+ if (!adetail)
+ opt.set_float("ip_adapter_strength", ip_adapter_strength);
+ if (!adetail)
+ opt.set_bool("hires.enabled", hires_enabled);
+ if (!adetail)
+ opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional<std::string>(resolved_hires_upscaler));
+ if (!adetail)
+ opt.set_string("hires.model_path", hires_upscaler_model_path);
+ if (!adetail)
+ opt.set_float("hires.scale", hires_scale);
+ if (!adetail)
+ opt.set_int("hires.target_width", hires_width);
+ if (!adetail)
+ opt.set_int("hires.target_height", hires_height);
+ if (!adetail)
+ opt.set_int("hires.steps", hires_steps);
+ if (!adetail)
+ opt.set_float("hires.denoising_strength", hires_denoising_strength);
+ if (!adetail)
+ opt.set_int("hires.upscale_tile_size", hires_upscale_tile_size);
+ if (circular)
+ opt.set_bool("circular_x", true);
+ else
+ opt.set_bool("circular_x", circular_x);
+ if (circular)
+ opt.set_bool("circular_y", true);
+ else
+ opt.set_bool("circular_y", circular_y);
+ opt.set_loras("loras", lora_vec.data(), lora_vec.size());
+ if (!adetail)
+ opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size());
+ if (!adetail)
+ opt.set_images("photomaker.id_images", pm_id_image_views.data(), pm_id_image_views.size());
+ if (!adetail)
+ opt.set_string("photomaker.id_embed_path", pm_id_embed_path);
+ if (!adetail)
+ opt.set_float("photomaker.style_strength", pm_style_strength);
+ if (!adetail)
+ opt.set_string("pulid.id_embedding_path", pulid_id_embedding_path);
+ if (!adetail)
+ opt.set_float("pulid.id_weight", pulid_id_weight);
+ if (!adetail)
+ opt.set_floats("hires.sigmas", hires_custom_sigmas);
+ if (!adetail)
+ opt.set_string("image_preprocess.rules", image_preprocess);
+ set_sample_options(opt, "sample.", sample_params, skip_layers, custom_sigmas, extra_sample_args);
+ set_cache_options(opt, cache_params);
+ set_tiling_options(opt, vae_tiling_params);
+ if (adetail) {
+ if (!ad_prompt.empty())
+ opt.set_string("adetail.prompt", ad_prompt);
+ if (!ad_negative_prompt.empty())
+ opt.set_string("adetail.negative_prompt", ad_negative_prompt);
+ opt.set_string("adetail.args", extra_ad_args);
+ }
+ return opt.finish();
}
-sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
- sd_vid_gen_params_t params;
- sd_vid_gen_params_init(¶ms);
-
+SDOptPtr SDGenerationParams::to_video_opt() {
lora_vec.clear();
lora_vec.reserve(lora_map.size() + high_noise_lora_map.size());
for (const auto& kv : lora_map) {
@@ -2985,7 +2856,7 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
for (auto& frame : ref_videos[i]) {
frame_views.push_back(frame.get());
}
- sd_audio_t audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : sd_audio_t{};
+ SDAudio audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : SDAudio{};
ref_video_views.push_back({frame_views.empty() ? nullptr : frame_views.data(),
static_cast<int>(frame_views.size()),
24,
@@ -2998,65 +2869,61 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() {
ref_audio_views.push_back(audio.get());
}
- sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data();
- sample_params.guidance.slg.layer_count = skip_layers.size();
- high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data();
- high_noise_sample_params.guidance.slg.layer_count = high_noise_skip_layers.size();
- sample_params.custom_sigmas = custom_sigmas.empty() ? nullptr : custom_sigmas.data();
- sample_params.custom_sigmas_count = static_cast<int>(custom_sigmas.size());
- sample_params.extra_sample_args = extra_sample_args.empty() ? nullptr : extra_sample_args.c_str();
- high_noise_sample_params.extra_sample_args = high_noise_extra_sample_args.empty() ? nullptr : high_noise_extra_sample_args.c_str();
- vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
- cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str();
-
- params.loras = lora_vec.empty() ? nullptr : lora_vec.data();
- params.lora_count = static_cast<uint32_t>(lora_vec.size());
- params.prompt = prompt.c_str();
- params.negative_prompt = negative_prompt.c_str();
- params.clip_skip = clip_skip;
- params.init_image = init_image.get();
- params.end_image = end_image.get();
- params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data();
- params.ref_images_count = static_cast<int>(ref_image_views.size());
- params.ref_videos = ref_video_views.empty() ? nullptr : ref_video_views.data();
- params.ref_videos_count = static_cast<int>(ref_video_views.size());
- params.ref_audios = ref_audio_views.empty() ? nullptr : ref_audio_views.data();
- params.ref_audios_count = static_cast<int>(ref_audio_views.size());
- params.control_frames = control_frame_views.empty() ? nullptr : control_frame_views.data();
- params.control_frames_size = static_cast<int>(control_frame_views.size());
- params.width = get_resolved_width();
- params.height = get_resolved_height();
- params.sample_params = sample_params;
- params.high_noise_sample_params = high_noise_sample_params;
- params.moe_boundary = moe_boundary;
- params.strength = strength;
- params.seed = seed;
- params.video_frames = video_frames;
- params.fps = fps;
- params.vace_strength = vace_strength;
- params.vae_tiling_params = vae_tiling_params;
- params.cache = cache_params;
- params.hires.enabled = hires_enabled;
- params.hires.upscaler = resolved_hires_upscaler;
- params.hires.model_path = hires_upscaler_model_path.empty() ? nullptr : hires_upscaler_model_path.c_str();
- params.hires.scale = hires_scale;
- params.hires.target_width = hires_width;
- params.hires.target_height = hires_height;
- params.hires.steps = hires_steps;
- params.hires.denoising_strength = hires_denoising_strength;
- params.hires.upscale_tile_size = hires_upscale_tile_size;
- params.hires.custom_sigmas = hires_custom_sigmas.empty() ? nullptr : hires_custom_sigmas.data();
- params.hires.custom_sigmas_count = static_cast<int>(hires_custom_sigmas.size());
- params.circular_x = circular || circular_x;
- params.circular_y = circular || circular_y;
- params.image_preprocess = {image_preprocess.c_str()};
- params.ref_image_args = ref_image_args.c_str();
- return params;
+ vae_tiling_params.extra_tiling_args = extra_tiling_args.empty() ? nullptr : extra_tiling_args.c_str();
+ cache_params.scm_mask = scm_mask.empty() ? nullptr : scm_mask.c_str();
+
+ SDOptBuilder opt(sd_vid_gen_opt_create());
+ opt.set_string("prompt", prompt);
+ opt.set_string("negative_prompt", negative_prompt);
+ opt.set_int("clip_skip", clip_skip);
+ opt.set_image("init_image", init_image.get());
+ opt.set_image("end_image", end_image.get());
+ opt.set_int("width", width);
+ opt.set_int("height", height);
+ opt.set_float("moe_boundary", moe_boundary);
+ opt.set_float("strength", strength);
+ opt.set_int("seed", seed);
+ opt.set_int("video_frames", video_frames);
+ opt.set_int("fps", fps);
+ opt.set_float("vace_strength", vace_strength);
+ opt.set_bool("hires.enabled", hires_enabled);
+ opt.set_string("hires.upscaler", resolved_hires_upscaler.empty() ? hires_upscaler : std::optional<std::string>(resolved_hires_upscaler));
+ opt.set_string("hires.model_path", hires_upscaler_model_path);
+ opt.set_float("hires.scale", hires_scale);
+ opt.set_int("hires.target_width", hires_width);
+ opt.set_int("hires.target_height", hires_height);
+ opt.set_int("hires.steps", hires_steps);
+ opt.set_float("hires.denoising_strength", hires_denoising_strength);
+ opt.set_int("hires.upscale_tile_size", hires_upscale_tile_size);
+ if (circular)
+ opt.set_bool("circular_x", true);
+ else
+ opt.set_bool("circular_x", circular_x);
+ if (circular)
+ opt.set_bool("circular_y", true);
+ else
+ opt.set_bool("circular_y", circular_y);
+ opt.set_string("ref_image_args", ref_image_args);
+ opt.set_loras("loras", lora_vec.data(), lora_vec.size());
+ opt.set_images("ref_images", ref_image_views.data(), ref_image_views.size());
+ opt.set_videos("ref_videos", ref_video_views.data(), ref_video_views.size());
+ opt.set_audios("ref_audios", ref_audio_views.data(), ref_audio_views.size());
+ opt.set_frames("control_frames", control_frame_views.data(), control_frame_views.size());
+ opt.set_floats("hires.sigmas", hires_custom_sigmas);
+ opt.set_string("image_preprocess.rules", image_preprocess);
+ opt.set_string("sample.scheduler", high_noise_sample_params.scheduler);
+ opt.set_float("sample.flow_shift", high_noise_sample_params.flow_shift);
+ opt.set_int("sample.shifted_timestep", high_noise_sample_params.shifted_timestep);
+ set_sample_options(opt, "sample.", sample_params, skip_layers, custom_sigmas, extra_sample_args);
+ set_sample_options(opt, "high_noise_sample.", high_noise_sample_params, high_noise_skip_layers, {}, high_noise_extra_sample_args);
+ set_cache_options(opt, cache_params);
+ set_tiling_options(opt, vae_tiling_params);
+ return opt.finish();
}
std::string SDGenerationParams::to_string() const {
- FreeUniquePtr<char> sample_params_str(sd_sample_params_to_str(&sample_params));
- FreeUniquePtr<char> high_noise_sample_params_str(sd_sample_params_to_str(&high_noise_sample_params));
+ const std::string sample_params_str = sample_params.sample_method.value_or("unset") + ", " + sample_params.scheduler.value_or("unset") + ", steps=" + (sample_params.sample_steps ? std::to_string(*sample_params.sample_steps) : "unset");
+ const std::string high_noise_sample_params_str = high_noise_sample_params.sample_method.value_or("unset") + ", " + high_noise_sample_params.scheduler.value_or("unset") + ", steps=" + (high_noise_sample_params.sample_steps ? std::to_string(*high_noise_sample_params.sample_steps) : "unset");
std::ostringstream lora_ss;
lora_ss << "{\n";
@@ -3113,15 +2980,15 @@ std::string SDGenerationParams::to_string() const {
<< " pm_id_images_dir: \"" << pm_id_images_dir << "\",\n"
<< " pm_id_embed_path: \"" << pm_id_embed_path << "\",\n"
<< " pm_style_strength: " << pm_style_strength << ",\n"
- << " skip_layers: " << vec_to_string(skip_layers) << ",\n"
- << " sample_params: " << SAFE_STR(sample_params_str.get()) << ",\n"
- << " high_noise_skip_layers: " << vec_to_string(high_noise_skip_layers) << ",\n"
- << " high_noise_sample_params: " << SAFE_STR(high_noise_sample_params_str.get()) << ",\n"
- << " custom_sigmas: " << vec_to_string(custom_sigmas) << ",\n"
+ << " skip_layers: " << (skip_layers ? vec_to_string(*skip_layers) : "unset") << ",\n"
+ << " sample_params: " << SAFE_STR(sample_params_str.c_str()) << ",\n"
+ << " high_noise_skip_layers: " << (high_noise_skip_layers ? vec_to_string(*high_noise_skip_layers) : "unset") << ",\n"
+ << " high_noise_sample_params: " << SAFE_STR(high_noise_sample_params_str.c_str()) << ",\n"
+ << " custom_sigmas: " << (custom_sigmas ? vec_to_string(*custom_sigmas) : "unset") << ",\n"
<< " cache_mode: \"" << cache_mode << "\",\n"
<< " cache_option: \"" << cache_option << "\",\n"
<< " cache: "
- << (cache_params.mode != SD_CACHE_DISABLED ? "enabled" : "disabled")
+ << cache_params.mode
<< " (threshold=" << cache_params.reuse_threshold
<< ", start=" << cache_params.start_percent
<< ", end=" << cache_params.end_percent << "),\n"
@@ -3135,7 +3002,7 @@ std::string SDGenerationParams::to_string() const {
<< " seed: " << seed << ",\n"
<< " upscale_repeats: " << upscale_repeats << ",\n"
<< " upscale_tile_size: " << upscale_tile_size << ",\n"
- << " hires: { enabled: " << (hires_enabled ? "true" : "false")
+ << " hires: { enabled: " << hires_enabled
<< ", upscaler: \"" << hires_upscaler << "\""
<< ", model_path: \"" << hires_upscaler_model_path << "\""
<< ", scale: " << hires_scale
@@ -3143,7 +3010,7 @@ std::string SDGenerationParams::to_string() const {
<< ", target_height: " << hires_height
<< ", steps: " << hires_steps
<< ", denoising_strength: " << hires_denoising_strength
- << ", custom_sigmas: " << vec_to_string(hires_custom_sigmas)
+ << ", custom_sigmas: " << (hires_custom_sigmas ? vec_to_string(*hires_custom_sigmas) : "unset")
<< ", upscale_tile_size: " << hires_upscale_tile_size << " },\n"
<< " vae_tiling_params: { "
<< vae_tiling_params.enabled << ", "
diff --git a/examples/common/common.h b/examples/common/common.h
index 43e11ef..eb4a043 100644
--- a/examples/common/common.h
+++ b/examples/common/common.h
@@ -2,6 +2,7 @@
#define __EXAMPLES_COMMON_COMMON_H__
#include "generation_metadata.h"
+#include "options.h"
#include <cmath>
#include <cstdint>
@@ -40,21 +41,21 @@ struct StringOption {
std::string long_name;
std::string desc;
int concat;
- std::string* target;
+ OptionTarget<std::string> target;
};
struct IntOption {
std::string short_name;
std::string long_name;
std::string desc;
- int* target;
+ OptionTarget<int> target;
};
struct FloatOption {
std::string short_name;
std::string long_name;
std::string desc;
- float* target;
+ OptionTarget<float> target;
};
struct BoolOption {
@@ -62,7 +63,7 @@ struct BoolOption {
std::string long_name;
std::string desc;
bool keep_true;
- bool* target;
+ OptionTarget<bool> target;
};
struct ManualFunction {
@@ -123,8 +124,8 @@ bool decode_base64_image(const std::string& encoded_input,
SDImageOwner& out_image);
struct SDContextParams {
- int n_threads = -1;
- int conditioning_cache_size = 4;
+ std::optional<int> n_threads;
+ std::optional<int> conditioning_cache_size;
std::string model_path;
std::string clip_l_path;
std::string clip_g_path;
@@ -138,7 +139,7 @@ struct SDContextParams {
std::string uncond_diffusion_model_path;
std::string embeddings_connectors_path;
std::string vae_path;
- std::string vae_format = "auto";
+ std::optional<std::string> vae_format;
std::string audio_vae_path;
std::string audio_encoder_path;
std::string taesd_path;
@@ -157,42 +158,41 @@ struct SDContextParams {
std::map<std::string, std::string> embedding_map;
std::vector<sd_embedding_t> embedding_vec;
- rng_type_t rng_type = CUDA_RNG;
- rng_type_t sampler_rng_type = RNG_TYPE_COUNT;
- bool offload_params_to_cpu = false;
- std::string max_vram = "0";
- bool disable_prefetch = false;
- bool disable_segmented_compute = false;
- bool eager_load = false;
- bool batched_cfg = true;
- std::string backend;
- std::string params_backend;
- std::string split_mode;
- std::string model_args;
- bool auto_fit = true;
- std::string rpc_servers;
+ std::optional<std::string> rng_type;
+ std::optional<std::string> sampler_rng_type;
+ bool offload_params_to_cpu = false;
+ std::optional<std::string> max_vram;
+ std::optional<bool> disable_prefetch;
+ std::optional<bool> disable_segmented_compute;
+ std::optional<bool> eager_load;
+ std::optional<bool> batched_cfg;
+ std::optional<std::string> backend;
+ std::optional<std::string> params_backend;
+ std::optional<std::string> split_mode;
+ std::optional<std::string> model_args;
+ std::optional<bool> auto_fit;
+ std::optional<std::string> rpc_servers;
bool list_devices = false;
std::string effective_backend;
std::string effective_params_backend;
- bool enable_mmap = false;
- bool control_net_cpu = false;
- bool clip_on_cpu = false;
- bool vae_on_cpu = false;
- std::string flash_attn_mode = "auto";
- std::string conv_direct_mode; // Empty selects auto while honoring legacy convolution flags.
- bool sage_attn = false;
- bool diffusion_conv_direct = false;
- bool vae_conv_direct = false;
- bool taesd_preview_only = false;
-
- prediction_t prediction = PREDICTION_COUNT;
- lora_apply_mode_t lora_apply_mode = LORA_APPLY_AUTO;
-
- bool force_sdxl_vae_conv_scale = false;
- float linear_scale = 0.f;
- float attn_scale = 0.f;
-
- float flow_shift = INFINITY;
+ std::optional<bool> enable_mmap;
+ bool control_net_cpu = false;
+ bool clip_on_cpu = false;
+ bool vae_on_cpu = false;
+ std::optional<std::string> flash_attn_mode;
+ std::optional<std::string> conv_direct_mode;
+ std::optional<bool> sage_attn;
+ std::optional<bool> diffusion_conv_direct;
+ std::optional<bool> vae_conv_direct;
+ std::optional<bool> taesd_preview_only;
+
+ std::optional<std::string> prediction;
+ std::optional<std::string> lora_apply_mode;
+
+ std::optional<bool> force_sdxl_vae_conv_scale;
+ std::optional<float> linear_scale;
+ std::optional<float> attn_scale;
+
ArgOptions get_options();
bool print_devices() const;
void build_embedding_map();
@@ -201,7 +201,7 @@ struct SDContextParams {
bool validate(SDMode mode);
bool resolve_and_validate(SDMode mode);
std::string to_string() const;
- sd_ctx_params_t to_sd_ctx_params_t();
+ SDOptPtr to_context_opt();
};
struct SDGenerationParams {
@@ -212,15 +212,15 @@ struct SDGenerationParams {
std::string ad_prompt;
std::string ad_negative_prompt;
std::string extra_ad_args;
- int clip_skip = -1; // <= 0 represents unspecified
- int width = -1;
- int height = -1;
- int batch_count = 1;
- int qwen_image_layers = 3;
- int64_t seed = 42;
- float strength = 0.75f;
- float control_strength = 0.9f;
- float ip_adapter_strength = 1.0f;
+ std::optional<int> clip_skip;
+ std::optional<int> width;
+ std::optional<int> height;
+ std::optional<int> batch_count;
+ std::optional<int> qwen_image_layers;
+ std::optional<int64_t> seed;
+ std::optional<float> strength;
+ std::optional<float> control_strength;
+ std::optional<float> ip_adapter_strength;
bool increase_ref_index = false;
bool embed_image_metadata = true;
@@ -228,7 +228,7 @@ struct SDGenerationParams {
std::string end_image_path;
std::string mask_image_path;
std::string control_image_path;
- std::string control_type;
+ std::optional<std::string> control_type;
std::string ip_adapter_image_path;
std::vector<std::string> ref_image_paths;
std::vector<std::string> ref_video_paths;
@@ -236,26 +236,26 @@ struct SDGenerationParams {
std::vector<std::string> ref_audio_paths;
std::string control_video_path;
- sd_sample_params_t sample_params;
- sd_sample_params_t high_noise_sample_params;
+ SDSampleParams sample_params;
+ SDSampleParams high_noise_sample_params;
std::string extra_sample_args;
std::string high_noise_extra_sample_args;
- std::vector<int> skip_layers = {7, 8, 9};
- std::vector<int> high_noise_skip_layers = {7, 8, 9};
+ std::optional<std::vector<int>> skip_layers;
+ std::optional<std::vector<int>> high_noise_skip_layers;
- std::vector<float> custom_sigmas;
+ std::optional<std::vector<float>> custom_sigmas;
std::string cache_mode;
std::string cache_option;
std::string scm_mask;
- bool scm_policy_dynamic = true;
- sd_cache_params_t cache_params{};
-
- float moe_boundary = 0.875f;
- int video_frames = 1;
- int fps = 16;
- float vace_strength = 1.f;
- sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
+ std::optional<bool> scm_policy_dynamic;
+ SDCacheParams cache_params{};
+
+ std::optional<float> moe_boundary;
+ std::optional<int> video_frames;
+ std::optional<int> fps;
+ std::optional<float> vace_strength;
+ SDTilingParams vae_tiling_params;
std::string extra_tiling_args;
std::string ref_image_args;
@@ -263,28 +263,28 @@ struct SDGenerationParams {
std::string pm_id_images_dir;
std::string pm_id_embed_path;
- float pm_style_strength = 20.f;
+ std::optional<float> pm_style_strength;
std::string pulid_id_embedding_path;
- float pulid_id_weight = 1.0f;
+ std::optional<float> pulid_id_weight;
- int upscale_repeats = 1;
- int upscale_tile_size = 128;
+ int upscale_repeats = 1;
+ std::optional<int> upscale_tile_size;
- bool circular = false;
- bool circular_x = false;
- bool circular_y = false;
+ bool circular = false;
+ std::optional<bool> circular_x;
+ std::optional<bool> circular_y;
- bool hires_enabled = false;
- std::string hires_upscaler = "Latent";
+ std::optional<bool> hires_enabled;
+ std::optional<std::string> hires_upscaler;
std::string hires_upscaler_model_path;
- float hires_scale = 2.f;
- int hires_width = 0;
- int hires_height = 0;
- int hires_steps = 0;
- float hires_denoising_strength = 0.7f;
- int hires_upscale_tile_size = 128;
- std::vector<float> hires_custom_sigmas;
+ std::optional<float> hires_scale;
+ std::optional<int> hires_width;
+ std::optional<int> hires_height;
+ std::optional<int> hires_steps;
+ std::optional<float> hires_denoising_strength;
+ std::optional<int> hires_upscale_tile_size;
+ std::optional<std::vector<float>> hires_custom_sigmas;
std::map<std::string, float> lora_map;
std::map<std::string, float> high_noise_lora_map;
@@ -292,8 +292,7 @@ struct SDGenerationParams {
// Derived and normalized fields.
std::string prompt_with_lora; // for metadata record only
std::vector<sd_lora_t> lora_vec;
- sd_hires_upscaler_t resolved_hires_upscaler;
- sd_control_type_t resolved_control_type = SD_CONTROL_TYPE_NONE;
+ std::string resolved_hires_upscaler;
// Owned execution payload.
SDImageOwner init_image;
@@ -309,13 +308,13 @@ struct SDGenerationParams {
std::vector<SDImageOwner> pm_id_images;
std::vector<SDImageOwner> control_frames;
- // Backing storage for sd_img_gen_params_t view fields.
- std::vector<sd_image_t> ref_image_views;
- std::vector<std::vector<sd_image_t>> ref_video_frame_views;
- std::vector<sd_ref_video_t> ref_video_views;
- std::vector<sd_audio_t> ref_audio_views;
- std::vector<sd_image_t> pm_id_image_views;
- std::vector<sd_image_t> control_frame_views;
+ // Media views used while populating the opaque options.
+ std::vector<SDImage> ref_image_views;
+ std::vector<std::vector<SDImage>> ref_video_frame_views;
+ std::vector<SDReferenceVideo> ref_video_views;
+ std::vector<SDAudio> ref_audio_views;
+ std::vector<SDImage> pm_id_image_views;
+ std::vector<SDImage> control_frame_views;
SDGenerationParams();
SDGenerationParams(const SDGenerationParams& other) = default;
@@ -331,16 +330,16 @@ struct SDGenerationParams {
bool extract_and_remove_lora(const std::string& lora_model_dir);
bool width_and_height_are_set() const;
void set_width_and_height_if_unset(int w, int h);
- int get_resolved_width() const;
- int get_resolved_height() const;
+ int get_requested_width() const;
+ int get_requested_height() const;
bool resolve(const std::string& lora_model_dir, const std::string& hires_upscalers_dir, bool strict = false, bool validate_missing_loras = false);
bool validate(SDMode mode);
bool resolve_and_validate(SDMode mode,
const std::string& lora_model_dir,
const std::string& hires_upscalers_dir,
bool strict = false);
- sd_img_gen_params_t to_sd_img_gen_params_t();
- sd_vid_gen_params_t to_sd_vid_gen_params_t();
+ SDOptPtr to_image_opt(bool adetail = false);
+ SDOptPtr to_video_opt();
std::string to_string() const;
};
diff --git a/examples/common/media_io.cpp b/examples/common/media_io.cpp
index e878c88..4159e46 100644
--- a/examples/common/media_io.cpp
+++ b/examples/common/media_io.cpp
@@ -409,7 +409,7 @@ bool extract_vp8_frame_from_webp(const std::vector<uint8_t>& webp_data, std::vec
return false;
}
-bool encode_sd_image_to_vp8_frame(const sd_image_t& image, int quality, std::vector<uint8_t>& vp8_frame) {
+bool encode_sd_image_to_vp8_frame(const SDImage& image, int quality, std::vector<uint8_t>& vp8_frame) {
if (image.data == nullptr || image.width == 0 || image.height == 0) {
return false;
}
@@ -659,7 +659,7 @@ void write_fourcc(std::vector<uint8_t>& data, const char* fourcc) {
data.insert(data.end(), fourcc, fourcc + 4);
}
-static std::vector<uint8_t> audio_to_pcm16_bytes(const sd_audio_t* audio) {
+static std::vector<uint8_t> audio_to_pcm16_bytes(const SDAudio* audio) {
if (audio == nullptr || audio->data == nullptr || audio->sample_count == 0 || audio->channels == 0 || audio->sample_rate == 0) {
return {};
}
@@ -674,7 +674,7 @@ static std::vector<uint8_t> audio_to_pcm16_bytes(const sd_audio_t* audio) {
return bytes;
}
-static std::pair<uint64_t, uint64_t> audio_sample_range_for_video_frame(const sd_audio_t* audio, int frame_idx, int num_frames, int fps) {
+static std::pair<uint64_t, uint64_t> audio_sample_range_for_video_frame(const SDAudio* audio, int frame_idx, int num_frames, int fps) {
if (audio == nullptr || fps <= 0 || num_frames <= 0) {
return {0, 0};
}
@@ -783,7 +783,7 @@ uint8_t* load_image_from_file(const char* image_path,
return load_image_common(false, image_path, 0, width, height, expected_width, expected_height, expected_channel, out_channel);
}
-bool load_sd_image_from_file(sd_image_t* image,
+bool load_sd_image_from_file(SDImage* image,
const char* image_path,
int expected_width,
int expected_height,
@@ -836,7 +836,7 @@ static void append_avi_metadata(std::vector<uint8_t>& data, const std::string& p
size_t start_pos = data.size();
}
-std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
+std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) {
if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n");
return {};
@@ -1043,7 +1043,7 @@ std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images
return avi_data;
}
-int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
+int create_mjpg_avi_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) {
std::vector<uint8_t> avi_data = create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
if (avi_data.empty()) {
return -1;
@@ -1056,7 +1056,7 @@ int create_mjpg_avi_from_sd_images(const char* filename, sd_image_t* images, int
}
#ifdef SD_USE_WEBP
-std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality) {
+std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality) {
if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n");
return {};
@@ -1102,7 +1102,7 @@ std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* i
int timestamp_ms = 0;
for (int i = 0; i < num_images; ++i) {
- const sd_image_t& image = images[i];
+ const SDImage& image = images[i];
if (static_cast<int>(image.width) != width || static_cast<int>(image.height) != height) {
fprintf(stderr, "Error: Frame dimensions do not match.\n");
return {};
@@ -1160,7 +1160,7 @@ std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* i
return std::vector<uint8_t>(webp_data.data.bytes, webp_data.data.bytes + webp_data.data.size);
}
-int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality) {
+int create_animated_webp_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality) {
std::vector<uint8_t> webp_data = create_animated_webp_from_sd_images_to_vector(images, num_images, fps, quality);
if (webp_data.empty()) {
return -1;
@@ -1174,7 +1174,7 @@ int create_animated_webp_from_sd_images(const char* filename, sd_image_t* images
#endif
#ifdef SD_USE_WEBM
-std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
+std::vector<uint8_t> create_webm_from_sd_images_to_vector(SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) {
if (num_images == 0) {
fprintf(stderr, "Error: Image array is empty.\n");
return {};
@@ -1261,7 +1261,7 @@ std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, in
uint64_t timestamp_ns = 0;
for (int i = 0; i < num_images; ++i) {
- const sd_image_t& image = images[i];
+ const SDImage& image = images[i];
if (static_cast<int>(image.width) != width || static_cast<int>(image.height) != height) {
fprintf(stderr, "Error: Frame dimensions do not match.\n");
return -1;
@@ -1314,7 +1314,7 @@ std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images, in
return writer.data();
}
-int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
+int create_webm_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) {
std::vector<uint8_t> webm_data = create_webm_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
if (webm_data.empty()) {
return -1;
@@ -1328,11 +1328,11 @@ int create_webm_from_sd_images(const char* filename, sd_image_t* images, int num
#endif
std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
- sd_image_t* images,
+ SDImage* images,
int num_images,
int fps,
int quality,
- const sd_audio_t* audio,
+ const SDAudio* audio,
const std::string& parameters) {
std::string format = output_format;
std::transform(format.begin(), format.end(), format.begin(),
@@ -1359,7 +1359,7 @@ std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& ou
return create_mjpg_avi_from_sd_images_to_vector(images, num_images, fps, quality, audio, parameters);
}
-int create_video_from_sd_images(const char* filename, sd_image_t* images, int num_images, int fps, int quality, const sd_audio_t* audio, const std::string& parameters) {
+int create_video_from_sd_images(const char* filename, SDImage* images, int num_images, int fps, int quality, const SDAudio* audio, const std::string& parameters) {
std::string path = filename ? filename : "";
auto pos = path.find_last_of('.');
std::string ext = pos == std::string::npos ? "" : path.substr(pos);
diff --git a/examples/common/media_io.h b/examples/common/media_io.h
index 7feb0ad..d765db7 100644
--- a/examples/common/media_io.h
+++ b/examples/common/media_io.h
@@ -5,7 +5,7 @@
#include <string>
#include <vector>
-#include "stable-diffusion.h"
+#include "resource_owners.hpp"
enum class EncodedImageFormat {
JPEG,
@@ -43,7 +43,7 @@ uint8_t* load_image_from_file(const char* image_path,
int expected_height = 0,
int expected_channel = 3);
-bool load_sd_image_from_file(sd_image_t* image,
+bool load_sd_image_from_file(SDImage* image,
const char* image_path,
int expected_width = 0,
int expected_height = 0,
@@ -59,26 +59,26 @@ uint8_t* load_image_from_memory(const char* image_bytes,
int expected_channel = 3);
int create_mjpg_avi_from_sd_images(const char* filename,
- sd_image_t* images,
+ SDImage* images,
int num_images,
int fps,
int quality = 90,
- const sd_audio_t* audio = nullptr,
+ const SDAudio* audio = nullptr,
const std::string& parameters = "");
-std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(sd_image_t* images,
+std::vector<uint8_t> create_mjpg_avi_from_sd_images_to_vector(SDImage* images,
int num_images,
int fps,
int quality = 90,
- const sd_audio_t* audio = nullptr,
+ const SDAudio* audio = nullptr,
const std::string& parameters = "");
#ifdef SD_USE_WEBP
int create_animated_webp_from_sd_images(const char* filename,
- sd_image_t* images,
+ SDImage* images,
int num_images,
int fps,
int quality = 90);
-std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* images,
+std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(SDImage* images,
int num_images,
int fps,
int quality = 90);
@@ -86,33 +86,33 @@ std::vector<uint8_t> create_animated_webp_from_sd_images_to_vector(sd_image_t* i
#ifdef SD_USE_WEBM
int create_webm_from_sd_images(const char* filename,
- sd_image_t* images,
+ SDImage* images,
int num_images,
int fps,
int quality = 90,
- const sd_audio_t* audio = nullptr,
+ const SDAudio* audio = nullptr,
const std::string& parameters = "");
-std::vector<uint8_t> create_webm_from_sd_images_to_vector(sd_image_t* images,
+std::vector<uint8_t> create_webm_from_sd_images_to_vector(SDImage* images,
int num_images,
int fps,
int quality = 90,
- const sd_audio_t* audio = nullptr,
+ const SDAudio* audio = nullptr,
const std::string& parameters = "");
#endif
int create_video_from_sd_images(const char* filename,
- sd_image_t* images,
+ SDImage* images,
int num_images,
int fps,
int quality = 90,
- const sd_audio_t* audio = nullptr,
+ const SDAudio* audio = nullptr,
const std::string& parameters = "");
std::vector<uint8_t> create_video_from_sd_images_to_vector(const std::string& output_format,
- sd_image_t* images,
+ SDImage* images,
int num_images,
int fps,
int quality = 90,
- const sd_audio_t* audio = nullptr,
+ const SDAudio* audio = nullptr,
const std::string& parameters = "");
bool write_wav_to_file(const std::string& path,
diff --git a/examples/common/options.h b/examples/common/options.h
new file mode 100644
index 0000000..3b86032
--- /dev/null
+++ b/examples/common/options.h
@@ -0,0 +1,185 @@
+#ifndef SD_EXAMPLES_OPTIONS_H
+#define SD_EXAMPLES_OPTIONS_H
+
+#include <cmath>
+#include <cstddef>
+#include <json.hpp>
+#include <optional>
+#include <ostream>
+#include <string>
+#include <type_traits>
+#include <utility>
+
+namespace nlohmann {
+ template <typename T>
+ struct adl_serializer<std::optional<T>> {
+ static void to_json(json& out, const std::optional<T>& value) {
+ out = value ? json(*value) : json(nullptr);
+ }
+ static void from_json(const json& in, std::optional<T>& value) {
+ value = in.is_null() ? std::nullopt : std::optional<T>(in.get<T>());
+ }
+ };
+}
+
+template <typename T>
+std::ostream& operator<<(std::ostream& out, const std::optional<T>& value) {
+ if (!value)
+ return out << "unset";
+ if constexpr (std::is_same_v<T, bool>)
+ return out << (*value ? "true" : "false");
+ else
+ return out << *value;
+}
+
+template <typename T>
+class OptionTarget {
+ T* value_ = nullptr;
+ std::optional<T>* input_ = nullptr;
+
+public:
+ OptionTarget(T* value)
+ : value_(value) {}
+ OptionTarget(std::optional<T>* input)
+ : input_(input) {}
+ T get() const { return input_ ? input_->value_or(T{}) : *value_; }
+ void set(T value) const {
+ if (input_)
+ *input_ = std::move(value);
+ else
+ *value_ = std::move(value);
+ }
+};
+
+template <typename T>
+struct InputType {
+ using type = T;
+};
+template <typename T>
+struct InputType<std::optional<T>> {
+ using type = T;
+};
+
+struct SDTilingParams {
+ std::optional<bool> enabled;
+ std::optional<bool> temporal_tiling;
+ std::optional<int> tile_size_w;
+ std::optional<int> tile_size_h;
+ std::optional<float> target_overlap;
+ std::optional<float> rel_size_w;
+ std::optional<float> rel_size_h;
+ const char* extra_tiling_args = nullptr;
+};
+
+struct SDSLGParams {
+ std::optional<float> layer_start;
+ std::optional<float> layer_end;
+ std::optional<float> scale;
+};
+
+struct SDGuidanceParams {
+ std::optional<float> txt_cfg;
+ std::optional<float> img_cfg;
+ std::optional<float> distilled_guidance;
+ SDSLGParams slg;
+};
+
+struct SDSampleParams {
+ SDGuidanceParams guidance;
+ std::optional<std::string> scheduler;
+ std::optional<std::string> sample_method;
+ std::optional<int> sample_steps;
+ std::optional<float> eta;
+ std::optional<int> shifted_timestep;
+ std::optional<float> flow_shift;
+};
+
+struct SDCacheParams {
+ std::optional<std::string> mode;
+ std::optional<float> reuse_threshold;
+ std::optional<float> start_percent;
+ std::optional<float> end_percent;
+ std::optional<float> error_decay_rate;
+ std::optional<bool> use_relative_threshold;
+ std::optional<bool> reset_error_on_compute;
+ std::optional<int> Fn_compute_blocks;
+ std::optional<int> Bn_compute_blocks;
+ std::optional<float> residual_diff_threshold;
+ std::optional<int> max_warmup_steps;
+ std::optional<int> max_cached_steps;
+ std::optional<int> max_continuous_cached_steps;
+ std::optional<int> taylorseer_n_derivatives;
+ std::optional<int> taylorseer_skip_interval;
+ const char* scm_mask = nullptr;
+ std::optional<bool> scm_policy_dynamic;
+ std::optional<float> spectrum_w;
+ std::optional<int> spectrum_m;
+ std::optional<float> spectrum_lam;
+ std::optional<int> spectrum_window_size;
+ std::optional<float> spectrum_flex_window;
+ std::optional<int> spectrum_warmup_steps;
+ std::optional<float> spectrum_stop_percent;
+};
+
+namespace example_options {
+ inline constexpr const char* sample_methods[] = {
+ "euler",
+ "euler_a",
+ "heun",
+ "dpm2",
+ "dpm++2s_a",
+ "dpm++2m",
+ "dpm++2mv2",
+ "ipndm",
+ "ipndm_v",
+ "lcm",
+ "ddim_trailing",
+ "tcd",
+ "res_multistep",
+ "res_2s",
+ "er_sde",
+ "euler_cfg_pp",
+ "euler_a_cfg_pp",
+ "euler_ge",
+ "dpm++2m_sde",
+ "dpm++2m_sde_bt",
+ "lms",
+ };
+
+ inline constexpr const char* schedulers[] = {
+ "discrete",
+ "karras",
+ "exponential",
+ "ays",
+ "gits",
+ "sgm_uniform",
+ "simple",
+ "smoothstep",
+ "kl_optimal",
+ "lcm",
+ "bong_tangent",
+ "ltx2",
+ "logit_normal",
+ "flux2",
+ "flux",
+ "beta",
+ "llada_image",
+ };
+
+ inline constexpr const char* preview_modes[] = {
+ "none",
+ "proj",
+ "tae",
+ "vae",
+ };
+
+ template <size_t N>
+ bool contains(const char* const (&values)[N], const std::string& value) {
+ for (const char* entry : values)
+ if (value == entry)
+ return true;
+ return false;
+ }
+}
+
+#endif
diff --git a/examples/common/resource_owners.hpp b/examples/common/resource_owners.hpp
index 8a8f843..8ba69e7 100644
--- a/examples/common/resource_owners.hpp
+++ b/examples/common/resource_owners.hpp
@@ -4,13 +4,136 @@
#include <cstdio>
#include <cstdlib>
#include <cstring>
+#include <limits>
#include <memory>
+#include <stdexcept>
+#include <string>
#include <utility>
#include <vector>
+#include <optional>
#include "generation_metadata.h"
+#include "log.h"
#include "stable-diffusion.h"
+struct SDImage {
+ uint32_t width;
+ uint32_t height;
+ uint32_t channel;
+ uint8_t* data;
+ sd_tensor_t* storage = nullptr;
+};
+
+struct SDAudio {
+ uint32_t sample_rate;
+ uint32_t channels;
+ uint64_t sample_count;
+ float* data;
+ sd_tensor_t* storage = nullptr;
+};
+
+struct SDReferenceVideo {
+ SDImage* frames;
+ int frame_count;
+ int fps;
+ SDAudio audio;
+};
+
+struct SDTensorDeleter {
+ void operator()(sd_tensor_t* tensor) const { sd_tensor_free(tensor); }
+};
+using SDTensorPtr = std::unique_ptr<sd_tensor_t, SDTensorDeleter>;
+
+static inline SDImage tensor_image_view(const sd_tensor_t* tensor) {
+ if (!tensor)
+ return {};
+ return {uint32_t(sd_tensor_dim(tensor, 1)), uint32_t(sd_tensor_dim(tensor, 2)), uint32_t(sd_tensor_dim(tensor, 0)),
+ const_cast<uint8_t*>(static_cast<const uint8_t*>(sd_tensor_data(tensor)))};
+}
+
+static inline void sd_image_free(SDImage* image) {
+ if (!image)
+ return;
+ if (image->storage)
+ sd_tensor_free(image->storage);
+ else
+ free(image->data);
+ *image = {};
+}
+
+static inline void free_sd_audio(SDAudio* audio) {
+ if (!audio)
+ return;
+ if (audio->storage)
+ sd_tensor_free(audio->storage);
+ else
+ free(audio->data);
+ free(audio);
+}
+
+static inline SDTensorPtr image_tensor(const SDImage& image) {
+ if (!image.data)
+ return {};
+ if (image.storage && sd_tensor_data(image.storage) == image.data &&
+ sd_tensor_dim(image.storage, 0) == image.channel && sd_tensor_dim(image.storage, 1) == image.width && sd_tensor_dim(image.storage, 2) == image.height)
+ return SDTensorPtr(sd_tensor_retain(image.storage));
+ const int64_t shape[] = {image.channel, image.width, image.height};
+ return SDTensorPtr(sd_tensor_create(SD_TENSOR_U8, 3, shape, image.data, size_t(image.width) * image.height * image.channel));
+}
+
+static inline SDTensorPtr audio_tensor(const SDAudio& audio) {
+ if (!audio.data)
+ return {};
+ if (audio.storage)
+ return SDTensorPtr(sd_tensor_retain(audio.storage));
+ const int64_t shape[] = {audio.channels, static_cast<int64_t>(audio.sample_count)};
+ return SDTensorPtr(sd_tensor_create(SD_TENSOR_F32, 2, shape, audio.data, size_t(audio.sample_count) * audio.channels * sizeof(float)));
+}
+
+static inline SDTensorPtr frames_tensor(const SDImage* frames, size_t count) {
+ if (!frames || !count)
+ return {};
+ const auto& first = frames[0];
+ const size_t frame_bytes = size_t(first.width) * first.height * first.channel;
+ if (!frame_bytes || count > SIZE_MAX / frame_bytes) {
+ LOG_ERROR("invalid video frame storage size");
+ return {};
+ }
+ std::vector<uint8_t> pixels(frame_bytes * count);
+ for (size_t i = 0; i < count; ++i) {
+ if (!frames[i].data || frames[i].width != first.width || frames[i].height != first.height || frames[i].channel != first.channel) {
+ LOG_ERROR("video frames must have matching dimensions and channels");
+ return {};
+ }
+ memcpy(pixels.data() + i * frame_bytes, frames[i].data, frame_bytes);
+ }
+ const int64_t shape[] = {first.channel, first.width, first.height, static_cast<int64_t>(count)};
+ return SDTensorPtr(sd_tensor_create(SD_TENSOR_U8, 4, shape, pixels.data(), pixels.size()));
+}
+
+static inline SDAudio* retain_result_audio(const sd_result_t* result) {
+ const auto* audio = sd_result_get_audio(result);
+ if (!audio)
+ return nullptr;
+ auto* view = static_cast<SDAudio*>(malloc(sizeof(SDAudio)));
+ if (!view)
+ return nullptr;
+ *view = {audio->sample_rate, uint32_t(sd_tensor_dim(audio->data, 0)), uint64_t(sd_tensor_dim(audio->data, 1)),
+ const_cast<float*>(static_cast<const float*>(sd_tensor_data(audio->data))), sd_tensor_retain(audio->data)};
+ return view;
+}
+
+static inline bool preprocess_canny(SDImage& image, float high, float low, float weak, float strong, bool inverse) {
+ auto input = image_tensor(image);
+ sd_tensor_t* output = nullptr;
+ if (!input || sd_preprocess_canny(input.get(), high, low, weak, strong, inverse, &output) != SD_STATUS_OK)
+ return false;
+ sd_image_free(&image);
+ image = tensor_image_view(output);
+ image.storage = output;
+ return true;
+}
+
struct FreeDeleter {
void operator()(void* ptr) const {
free(ptr);
@@ -28,23 +151,23 @@ struct FileCloser {
struct SDCtxDeleter {
void operator()(sd_ctx_t* ctx) const {
if (ctx != nullptr) {
- free_sd_ctx(ctx);
+ sd_ctx_free(ctx);
}
}
};
struct UpscalerCtxDeleter {
- void operator()(upscaler_ctx_t* ctx) const {
+ void operator()(sd_upscaler_t* ctx) const {
if (ctx != nullptr) {
- free_upscaler_ctx(ctx);
+ sd_upscaler_free(ctx);
}
}
};
struct ADetailerCtxDeleter {
- void operator()(adetailer_ctx_t* ctx) const {
+ void operator()(sd_adetailer_t* ctx) const {
if (ctx != nullptr) {
- free_adetailer_ctx(ctx);
+ sd_adetailer_free(ctx);
}
}
};
@@ -54,12 +177,280 @@ using FreeUniquePtr = std::unique_ptr<T, FreeDeleter>;
using FilePtr = std::unique_ptr<FILE, FileCloser>;
using SDCtxPtr = std::unique_ptr<sd_ctx_t, SDCtxDeleter>;
-using UpscalerCtxPtr = std::unique_ptr<upscaler_ctx_t, UpscalerCtxDeleter>;
-using ADetailerCtxPtr = std::unique_ptr<adetailer_ctx_t, ADetailerCtxDeleter>;
+using UpscalerCtxPtr = std::unique_ptr<sd_upscaler_t, UpscalerCtxDeleter>;
+using ADetailerCtxPtr = std::unique_ptr<sd_adetailer_t, ADetailerCtxDeleter>;
+
+struct SDOptDeleter {
+ void operator()(sd_opt_t* opt) const { sd_opt_free(opt); }
+};
+struct SDResultDeleter {
+ void operator()(sd_result_t* result) const { sd_result_free(result); }
+};
+using SDOptPtr = std::unique_ptr<sd_opt_t, SDOptDeleter>;
+using SDResultPtr = std::unique_ptr<sd_result_t, SDResultDeleter>;
+
+class SDOptBuilder {
+ SDOptPtr opt;
+ sd_status_t status;
+
+public:
+ explicit SDOptBuilder(sd_opt_t* value)
+ : opt(value), status(value ? SD_STATUS_OK : SD_STATUS_OUT_OF_MEMORY) {}
+ SDOptPtr finish() { return status == SD_STATUS_OK ? std::move(opt) : SDOptPtr{}; }
+ void unset(const char* key) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_unset(opt.get(), key);
+ }
+ void set_floats(const char* key, const std::optional<std::vector<float>>& value) {
+ if (value)
+ set_floats(key, value->data(), value->size());
+ }
+ void set_bool(const char* key, const std::optional<bool>& value) {
+ if (value)
+ set_bool(key, *value);
+ }
+ template <typename T>
+ void set_int(const char* key, const std::optional<T>& value) {
+ if (value)
+ set_int(key, *value);
+ }
+ template <typename T>
+ void set_float(const char* key, const std::optional<T>& value) {
+ if (value)
+ set_float(key, *value);
+ }
+ void set_string(const char* key, const std::optional<std::string>& value) {
+ if (value)
+ set_string(key, *value);
+ }
+ void set_bool(const char* key, bool value) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_set_bool(opt.get(), key, value);
+ }
+ void set_int(const char* key, int64_t value) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_set_int(opt.get(), key, value);
+ }
+ void set_float(const char* key, double value) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_set_float(opt.get(), key, value);
+ }
+ void set_string(const char* key, const std::string& value) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_set_string(opt.get(), key, value.c_str());
+ }
+ void set_string(const char* key, const char* value) {
+ if (value && status == SD_STATUS_OK)
+ status = sd_opt_set_string(opt.get(), key, value);
+ }
+ void set_image(const char* key, const SDImage& value) {
+ if (!value.data || status != SD_STATUS_OK)
+ return;
+ auto tensor = image_tensor(value);
+ if (!tensor) {
+ status = SD_STATUS_INVALID_ARGUMENT;
+ return;
+ }
+ status = sd_opt_set_tensor(opt.get(), key, tensor.get());
+ }
+ void set_images(const char* key, const SDImage* value, size_t count) {
+ if (status != SD_STATUS_OK)
+ return;
+ std::vector<SDTensorPtr> storage;
+ std::vector<const sd_tensor_t*> tensors;
+ storage.reserve(count);
+ tensors.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ storage.push_back(image_tensor(value[i]));
+ if (!storage.back()) {
+ status = SD_STATUS_INVALID_ARGUMENT;
+ return;
+ }
+ tensors.push_back(storage.back().get());
+ }
+ status = sd_opt_set_tensors(opt.get(), key, tensors.data(), count);
+ }
+ void set_frames(const char* key, const SDImage* value, size_t count) {
+ if (!count || status != SD_STATUS_OK)
+ return;
+ auto tensor = frames_tensor(value, count);
+ if (!tensor) {
+ status = SD_STATUS_INVALID_ARGUMENT;
+ return;
+ }
+ status = sd_opt_set_tensor(opt.get(), key, tensor.get());
+ }
+ void set_floats(const char* key, const float* value, size_t count) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_set_floats(opt.get(), key, value, count);
+ }
+ void set_ints(const char* key, const int32_t* value, size_t count) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_set_ints(opt.get(), key, value, count);
+ }
+ void set_loras(const char* key, const sd_lora_t* value, size_t count) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_set_loras(opt.get(), key, value, count);
+ }
+ void set_embeddings(const char* key, const sd_embedding_t* value, size_t count) {
+ if (status == SD_STATUS_OK)
+ status = sd_opt_set_embeddings(opt.get(), key, value, count);
+ }
+ void set_audios(const char* key, const SDAudio* value, size_t count) {
+ if (status != SD_STATUS_OK)
+ return;
+ std::vector<SDTensorPtr> storage;
+ std::vector<sd_audio_t> audios;
+ storage.reserve(count);
+ audios.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ storage.push_back(audio_tensor(value[i]));
+ audios.push_back({value[i].sample_rate, storage.back().get()});
+ }
+ status = sd_opt_set_audios(opt.get(), key, audios.data(), count);
+ }
+ void set_videos(const char* key, const SDReferenceVideo* value, size_t count) {
+ if (status != SD_STATUS_OK)
+ return;
+ std::vector<SDTensorPtr> storage;
+ std::vector<sd_video_t> videos;
+ storage.reserve(count * 2);
+ videos.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ storage.push_back(frames_tensor(value[i].frames, value[i].frame_count));
+ const auto* frames = storage.back().get();
+ storage.push_back(audio_tensor(value[i].audio));
+ videos.push_back({frames, value[i].fps, {value[i].audio.sample_rate, storage.back().get()}});
+ }
+ status = sd_opt_set_videos(opt.get(), key, videos.data(), count);
+ }
+};
+
+static inline sd_ctx_t* create_sd_context(const SDOptPtr& opt) {
+ sd_ctx_t* ctx = nullptr;
+ if (opt)
+ sd_ctx_create(opt.get(), &ctx);
+ return ctx;
+}
+
+static inline bool generate_sd_image(sd_ctx_t* ctx, const SDOptPtr& opt, sd_result_t** result) {
+ if (result)
+ *result = nullptr;
+ return opt && sd_generate_image(ctx, opt.get(), result) == SD_STATUS_OK;
+}
+
+static inline bool generate_sd_video(sd_ctx_t* ctx, const SDOptPtr& opt, sd_result_t** result) {
+ if (result)
+ *result = nullptr;
+ return opt && sd_generate_video(ctx, opt.get(), result) == SD_STATUS_OK;
+}
+
+static inline sd_upscaler_t* create_sd_upscaler(const char* path, std::optional<bool> direct, std::optional<int> threads, std::optional<int> tile, const char* backend, const char* params_backend) {
+ SDOptPtr opt(sd_upscaler_opt_create());
+ if (!opt || sd_opt_set_string(opt.get(), "model_path", path) != SD_STATUS_OK ||
+ (direct && sd_opt_set_bool(opt.get(), "conv_direct", *direct) != SD_STATUS_OK) ||
+ (tile && sd_opt_set_int(opt.get(), "tile_size", *tile) != SD_STATUS_OK) ||
+ (threads && sd_opt_set_int(opt.get(), "n_threads", *threads) != SD_STATUS_OK) ||
+ (backend && sd_opt_set_string(opt.get(), "backend", backend) != SD_STATUS_OK) ||
+ (params_backend && sd_opt_set_string(opt.get(), "params_backend", params_backend) != SD_STATUS_OK))
+ return nullptr;
+ sd_upscaler_t* result = nullptr;
+ sd_upscaler_create(opt.get(), &result);
+ return result;
+}
+
+static inline bool upscale_sd_image(sd_upscaler_t* ctx, SDImage image, sd_result_t** result) {
+ if (result)
+ *result = nullptr;
+ SDOptPtr opt(sd_upscale_opt_create());
+ auto tensor = image_tensor(image);
+ return opt && tensor && sd_opt_set_tensor(opt.get(), "image", tensor.get()) == SD_STATUS_OK &&
+ sd_upscale(ctx, opt.get(), result) == SD_STATUS_OK;
+}
+
+static inline sd_adetailer_t* create_sd_adetailer(const char* path, std::optional<int> threads, const char* backend, const char* params_backend) {
+ SDOptPtr opt(sd_adetailer_opt_create());
+ if (!opt || sd_opt_set_string(opt.get(), "model_path", path) != SD_STATUS_OK ||
+ (threads && sd_opt_set_int(opt.get(), "n_threads", *threads) != SD_STATUS_OK) ||
+ (backend && sd_opt_set_string(opt.get(), "backend", backend) != SD_STATUS_OK) ||
+ (params_backend && sd_opt_set_string(opt.get(), "params_backend", params_backend) != SD_STATUS_OK))
+ return nullptr;
+ sd_adetailer_t* result = nullptr;
+ sd_adetailer_create(opt.get(), &result);
+ return result;
+}
+
+static inline bool adetail_sd_image(sd_adetailer_t* detector, sd_ctx_t* ctx, SDImage image, SDOptPtr opt, sd_result_t** result) {
+ if (result)
+ *result = nullptr;
+ auto tensor = image_tensor(image);
+ return opt && tensor && sd_opt_set_tensor(opt.get(), "image", tensor.get()) == SD_STATUS_OK &&
+ sd_adetail(detector, ctx, opt.get(), result) == SD_STATUS_OK;
+}
+
+struct SDVideoUpscalerParams {
+ const char* diffusion_model_path{};
+ const char* vae_path{};
+ // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints.
+ int upscale_factor{};
+ std::optional<int> n_threads;
+ const char* backend{};
+ const char* params_backend{};
+ // auto/on/off or diffusion/vae assignments; null selects auto.
+ const char* flash_attn_mode{};
+ const char* conv_direct_mode{};
+};
+
+struct SDVideoUpscaleParams {
+ const SDImage* frames{};
+ int frame_count{};
+ std::optional<int> fps;
+ std::optional<int64_t> seed;
+ // RelayVSR reference window, including both endpoints; defaults to 16.
+ std::optional<int> window_size;
+ // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5.
+ std::optional<int> temporal_chunk_size;
+};
+
+static inline sd_vid_upscaler_t* create_sd_vid_upscaler(const SDVideoUpscalerParams& config) {
+ SDOptPtr opt(sd_vid_upscaler_opt_create());
+ if (!opt || sd_opt_set_string(opt.get(), "diffusion_model_path", config.diffusion_model_path) != SD_STATUS_OK ||
+ sd_opt_set_string(opt.get(), "vae_path", config.vae_path) != SD_STATUS_OK ||
+ sd_opt_set_int(opt.get(), "upscale_factor", config.upscale_factor) != SD_STATUS_OK ||
+ (config.n_threads && sd_opt_set_int(opt.get(), "n_threads", *config.n_threads) != SD_STATUS_OK) ||
+ (config.backend && sd_opt_set_string(opt.get(), "backend", config.backend) != SD_STATUS_OK) ||
+ (config.params_backend && sd_opt_set_string(opt.get(), "params_backend", config.params_backend) != SD_STATUS_OK) ||
+ (config.flash_attn_mode && sd_opt_set_string(opt.get(), "flash_attn_mode", config.flash_attn_mode) != SD_STATUS_OK) ||
+ (config.conv_direct_mode && sd_opt_set_string(opt.get(), "conv_direct_mode", config.conv_direct_mode) != SD_STATUS_OK))
+ return nullptr;
+ sd_vid_upscaler_t* result = nullptr;
+ sd_vid_upscaler_create(opt.get(), &result);
+ return result;
+}
+
+static inline bool upscale_sd_video(sd_vid_upscaler_t* ctx, const SDVideoUpscaleParams& config, sd_result_t** result) {
+ if (result)
+ *result = nullptr;
+ SDOptBuilder opt(sd_vid_upscale_opt_create());
+ opt.set_frames("frames", config.frames, config.frame_count);
+ opt.set_int("fps", config.fps);
+ opt.set_int("seed", config.seed);
+ opt.set_int("window_size", config.window_size);
+ opt.set_int("temporal_chunk_size", config.temporal_chunk_size);
+ auto params = opt.finish();
+ return params && sd_upscale_video(ctx, params.get(), result) == SD_STATUS_OK;
+}
+
+static inline SDImage take_result_image(sd_result_t* result, size_t index) {
+ const auto* tensor = sd_result_get_image(result, index);
+ auto image = tensor_image_view(tensor);
+ image.storage = sd_tensor_retain(tensor);
+ return image;
+}
class SDImageOwner {
private:
- static sd_image_t copy_image(const sd_image_t& image) {
+ static SDImage copy_image(const SDImage& image) {
if (image.data == nullptr) {
return {image.width, image.height, image.channel, nullptr};
}
@@ -74,11 +465,12 @@ private:
return {image.width, image.height, image.channel, raw_copy};
}
- sd_image_t image_ = {0, 0, 0, nullptr};
+ SDImage image_ = {0, 0, 0, nullptr};
+ SDTensorPtr public_tensor_;
public:
SDImageOwner() = default;
- explicit SDImageOwner(sd_image_t image)
+ explicit SDImageOwner(SDImage image)
: image_(image) {
}
@@ -109,9 +501,14 @@ public:
reset();
}
- sd_image_t* put() {
+ const sd_tensor_t* tensor() {
+ public_tensor_ = image_tensor(image_);
+ return public_tensor_.get();
+ }
+
+ SDImage* put() {
if (image_.data != nullptr) {
- free(image_.data);
+ sd_image_free(&image_);
image_.data = nullptr;
}
image_.width = 0;
@@ -120,23 +517,23 @@ public:
return &image_;
}
- sd_image_t& get() {
+ SDImage& get() {
return image_;
}
- const sd_image_t& get() const {
+ const SDImage& get() const {
return image_;
}
- sd_image_t release() {
- sd_image_t image = image_;
- image_ = {0, 0, 0, nullptr};
+ SDImage release() {
+ SDImage image = image_;
+ image_ = {0, 0, 0, nullptr};
return image;
}
- void reset(sd_image_t image = {0, 0, 0, nullptr}) {
+ void reset(SDImage image = {0, 0, 0, nullptr}) {
if (image_.data != nullptr) {
- free(image_.data);
+ sd_image_free(&image_);
}
image_ = image;
}
@@ -161,7 +558,7 @@ public:
return samples_.empty();
}
- sd_audio_t get() {
+ SDAudio get() {
return {sample_rate_,
channels_,
channels_ == 0 ? 0 : static_cast<uint64_t>(samples_.size() / channels_),
@@ -175,7 +572,7 @@ public:
class SDImageVec {
private:
- std::vector<sd_image_t> images_;
+ std::vector<SDImage> images_;
std::vector<std::string> generation_infos_;
public:
@@ -201,7 +598,7 @@ public:
clear();
}
- void push_back(sd_image_t image) {
+ void push_back(SDImage image) {
images_.push_back(image);
}
@@ -223,32 +620,25 @@ public:
generation_infos_.at(index) = merge_generation_info(generation_infos_.at(index), generation_info_with_relative_lora_paths(info, lora_model_dir));
}
- void adopt(sd_image_generation_result_t* result, const std::string& lora_model_dir = {}) {
+ void adopt(sd_result_t* result, const std::string& lora_model_dir = {}) {
clear();
+ SDResultPtr owner(result);
if (!result)
return;
- for (int i = 0; i < result->count; ++i) {
- images_.push_back(result->items[i].image);
- result->items[i].image = {};
- generation_infos_.push_back(generation_info_with_relative_lora_paths(result->items[i].generation_info, lora_model_dir));
+ const size_t count = sd_result_get_image_count(result);
+ images_.reserve(count);
+ generation_infos_.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ generation_infos_.push_back(generation_info_with_relative_lora_paths(sd_result_get_generation_info(result, i), lora_model_dir));
+ images_.push_back(take_result_image(result, i));
}
- free_sd_image_generation_result(result);
}
- void adopt_video(sd_video_generation_result_t* result, const std::string& lora_model_dir = {}) {
- clear();
- if (!result)
- return;
- const auto info = generation_info_with_relative_lora_paths(result->generation_info, lora_model_dir);
- for (int i = 0; i < result->frame_count; ++i) {
- images_.push_back(result->frames[i]);
- result->frames[i] = {};
- generation_infos_.push_back(info);
- }
- free_sd_video_generation_result(result);
+ void adopt_video(sd_result_t* result, const std::string& lora_model_dir = {}) {
+ adopt(result, lora_model_dir);
}
- void adopt(sd_image_t* images, int count) {
+ void adopt(SDImage* images, int count) {
clear();
if (images == nullptr || count <= 0) {
free(images);
@@ -278,33 +668,33 @@ public:
return !images_.empty();
}
- sd_image_t* data() {
+ SDImage* data() {
return images_.data();
}
- const sd_image_t* data() const {
+ const SDImage* data() const {
return images_.data();
}
- sd_image_t& operator[](size_t index) {
+ SDImage& operator[](size_t index) {
return images_[index];
}
- const sd_image_t& operator[](size_t index) const {
+ const SDImage& operator[](size_t index) const {
return images_[index];
}
- std::vector<sd_image_t>& raw() {
+ std::vector<SDImage>& raw() {
return images_;
}
- const std::vector<sd_image_t>& raw() const {
+ const std::vector<SDImage>& raw() const {
return images_;
}
void clear() {
- for (sd_image_t& image : images_) {
- free(image.data);
+ for (SDImage& image : images_) {
+ sd_image_free(&image);
image.data = nullptr;
}
images_.clear();
diff --git a/examples/server/README.md b/examples/server/README.md
index 6b6e750..8673eb8 100644
--- a/examples/server/README.md
+++ b/examples/server/README.md
@@ -10,6 +10,16 @@ and HTTP APIs. Download or build the server before starting; see the
- [API](#api): native and compatibility endpoints.
- [Frontend development](#frontend-development): build or customize the Web UI.
+Inference settings follow this priority: request fields, explicit server startup
+options, model recipe presets, then [library defaults](../../docs/guides/c_api.md#keys).
+Omitted request fields retain the configured override. In native JSON requests,
+`null` clears an optional inference field so the library selects its value;
+`0`, `false`, empty strings and empty numeric arrays are explicit values.
+Capability `defaults` and `defaults_by_mode` contain configured overrides, with
+`null` for optional settings left to the library. Effective settings are returned
+in generation metadata. Output formats, queue limits and listen addresses keep
+server-local defaults.
+
## Web UI
### Start the server
diff --git a/examples/server/api.md b/examples/server/api.md
index dca1351..9f48e24 100644
--- a/examples/server/api.md
+++ b/examples/server/api.md
@@ -229,6 +229,9 @@ values reflect these declarations. A loaded recipe is snapshotted: refreshing
recipe files does not change the active context's capabilities until reloading.
Command-line preloading uses the runtime-only all-features recipe instead of model-family inference
and component paths; variant-specific recipe declarations are more precise.
+Known high-noise sampling settings and `moe_boundary` are ignored when no
+high-noise stage is loaded, and `vace_strength` is ignored by non-VACE models.
+Actual control-frame inputs still require VACE.
These declarations govern the Web UI and advertised capabilities, not API
request validation. Existing generation APIs continue accepting their existing
@@ -1282,7 +1285,7 @@ and `ref_images[]`. RGB-only models drop it, so sending RGBA is safe for every m
If omitted or null:
-- single-image fields map to an empty `sd_image_t`
+- single-image fields clear the corresponding image tensor
- array fields map to an empty C-style array, represented as `pointer = nullptr` and `count = 0`
### Field Mapping Summary
@@ -1645,7 +1648,7 @@ Frame ordering rules:
If omitted or null:
-- single-image fields map to an empty `sd_image_t`
+- single-image fields clear the corresponding image tensor
- array fields map to an empty C-style array, represented as `pointer = nullptr` and `count = 0`
### Field Mapping Summary
@@ -1707,6 +1710,13 @@ Sampling fields:
High-noise sampling fields:
+The two video stages share the scheduler, flow shift and timestep shifting.
+`high_noise_sample_params.scheduler`, `.flow_shift` and `.shifted_timestep`
+are accepted as aliases for those shared settings in video requests. Image
+requests ignore these high-noise fields. Within one video request, a
+corresponding `sample_params` field takes precedence. Request aliases override
+inherited startup and recipe values; `null` clears the inherited shared setting.
+
| Field | Type |
| --- | --- |
| `high_noise_sample_params.scheduler` | `string` |
diff --git a/examples/server/async_jobs.cpp b/examples/server/async_jobs.cpp
index 3513c4a..bb88fb6 100644
--- a/examples/server/async_jobs.cpp
+++ b/examples/server/async_jobs.cpp
@@ -43,14 +43,18 @@ public:
}
};
-static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) {
+static void preview_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) {
+ std::vector<SDImage> views;
+ for (int i = 0; i < frame_count; ++i)
+ views.push_back(tensor_image_view(tensors[i]));
+ SDImage* frames = views.data();
(void)is_noisy;
if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) {
return;
}
AsyncGenerationJob* job = static_cast<AsyncGenerationJob*>(data);
- const sd_image_t& frame = frames[0];
+ const SDImage& frame = frames[0];
const sd_preview_info_t info = sd_get_preview_info();
auto image_bytes = encode_image_to_vector(EncodedImageFormat::PNG,
@@ -73,23 +77,20 @@ static void preview_callback(int step, int frame_count, sd_image_t* frames, bool
}
static void clear_preview_callback() {
- sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr);
+ sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr);
}
static void set_preview_callback(AsyncGenerationJob& job) {
- preview_t mode = str_to_preview(job.kind == AsyncJobKind::ImgGen
- ? job.img_gen.preview_mode.c_str()
- : job.vid_gen.preview_mode.c_str());
- if (mode == PREVIEW_COUNT || mode == PREVIEW_NONE) {
+ const auto& mode = job.kind == AsyncJobKind::ImgGen ? job.img_gen.preview_mode : job.vid_gen.preview_mode;
+ if (mode == "none")
return;
- }
int interval = job.kind == AsyncJobKind::ImgGen
? job.img_gen.preview_interval
: job.vid_gen.preview_interval;
if (interval <= 0) {
interval = 1;
}
- sd_set_preview_callback(preview_callback, mode, interval, true, false, &job);
+ sd_set_preview_callback(preview_callback, mode.c_str(), interval, true, false, &job);
}
static void update_job_progress(AsyncGenerationJob& job, const char* stage, const char* detail = "", int sample_pass = -1, int64_t completed = 0, int64_t total = 0, const char* unit = "") {
@@ -343,8 +344,8 @@ bool execute_img_gen_job(ServerRuntime& runtime,
std::vector<std::string>& output_images,
json& output_info,
std::string& error_message) {
- output_info = json::array();
- sd_img_gen_params_t params = job.img_gen.to_sd_img_gen_params_t();
+ output_info = json::array();
+ SDOptPtr params = job.img_gen.to_image_opt();
SDImageVec results;
@@ -356,15 +357,15 @@ bool execute_img_gen_job(ServerRuntime& runtime,
JobCancellationMonitor cancellation(runtime.sd_ctx, job);
JobCallbacksGuard callbacks(runtime.sd_ctx, job);
PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.img_gen.gen_params);
- sd_image_generation_result_t* generated = nullptr;
- generate_image(runtime.sd_ctx, ¶ms, &generated);
+ sd_result_t* generated = nullptr;
+ generate_sd_image(runtime.sd_ctx, params, &generated);
results.adopt(generated, runtime.ctx_params->lora_model_dir);
if (job.cancel_requested.load()) {
return false;
}
if (!results.empty() && job.img_gen.adetailer_enabled && !runtime.default_gen_params->ad_model_path.empty()) {
callbacks.begin_detailing();
- if (!apply_server_adetailer(runtime, job.img_gen, params, results)) {
+ if (!apply_server_adetailer(runtime, job.img_gen, results)) {
error_message = "ADetailer failed";
return false;
}
@@ -424,12 +425,12 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
int& output_fps,
json& output_info,
std::string& error_message) {
- sd_vid_gen_params_t params = job.vid_gen.to_sd_vid_gen_params_t();
+ SDOptPtr params = job.vid_gen.to_video_opt();
std::string str_params;
SDImageVec results;
- int num_results = 0;
- sd_audio_t* generated_audio = nullptr;
+ int num_results = 0;
+ SDAudio* generated_audio = nullptr;
{
std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
@@ -439,11 +440,11 @@ bool execute_vid_gen_job(ServerRuntime& runtime,
JobCancellationMonitor cancellation(runtime.sd_ctx, job);
JobCallbacksGuard callbacks(runtime.sd_ctx, job);
PromptImageCallbackGuard prompt_images(runtime.sd_ctx, job.vid_gen.gen_params);
- sd_video_generation_result_t* generated = nullptr;
- if (generate_video(runtime.sd_ctx, ¶ms, &generated)) {
- output_fps = generated->fps;
- generated_audio = generated->audio;
- generated->audio = nullptr;
+ sd_result_t* generated = nullptr;
+ if (generate_sd_video(runtime.sd_ctx, params, &generated)) {
+ output_fps = sd_result_get_fps(generated);
+ generated_audio = retain_result_audio(generated);
+
results.adopt_video(generated, runtime.ctx_params->lora_model_dir);
output_info = json::parse(results.generation_info(0));
if (job.vid_gen.gen_params.embed_image_metadata)
diff --git a/examples/server/main.cpp b/examples/server/main.cpp
index 67d557a..391a1ec 100644
--- a/examples/server/main.cpp
+++ b/examples/server/main.cpp
@@ -52,8 +52,6 @@ static void parse_args(int argc,
exit(ctx_params.print_devices() ? 0 : 1);
}
- const bool random_seed_requested = default_gen_params.seed < 0;
-
if (!svr_params.resolve_and_validate() ||
!ctx_params.resolve_and_validate(ctx_params.model_path.empty() && ctx_params.diffusion_model_path.empty() ? METADATA : IMG_GEN) ||
!default_gen_params.resolve_and_validate(IMG_GEN,
@@ -62,10 +60,6 @@ static void parse_args(int argc,
print_usage(argv[0], options_vec);
exit(1);
}
-
- if (random_seed_requested) {
- default_gen_params.seed = -1;
- }
}
void sd_log_cb(enum sd_log_level_t level, const char* log, void* data) {
@@ -93,16 +87,16 @@ int main(int argc, const char** argv) {
LOG_DEBUG("%s", ctx_params.to_string().c_str());
LOG_DEBUG("%s", default_gen_params.to_string().c_str());
- sd_ctx_params_t sd_ctx_params = ctx_params.to_sd_ctx_params_t();
+ SDOptPtr sd_ctx_params = ctx_params.to_context_opt();
ServerModels models;
if (!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) {
const auto started = std::chrono::steady_clock::now();
- models.context.reset(new_sd_ctx(&sd_ctx_params));
+ models.context.reset(create_sd_context(sd_ctx_params));
models.status["load_elapsed_seconds"] = std::chrono::duration<double>(std::chrono::steady_clock::now() - started).count();
}
if ((!ctx_params.model_path.empty() || !ctx_params.diffusion_model_path.empty()) && !models.context) {
- LOG_ERROR("new_sd_ctx_t failed");
+ LOG_ERROR("sd_ctx_create failed");
return 1;
}
diff --git a/examples/server/model_catalog.cpp b/examples/server/model_catalog.cpp
index f665567..5a538fd 100644
--- a/examples/server/model_catalog.cpp
+++ b/examples/server/model_catalog.cpp
@@ -193,19 +193,26 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para
throw std::runtime_error(std::string(key) + " is out of range");
}
if (std::string(key) == "width")
- params.width = static_cast<int>(value);
+ if (!params.width)
+ params.width = static_cast<int>(value);
if (std::string(key) == "height")
- params.height = static_cast<int>(value);
+ if (!params.height)
+ params.height = static_cast<int>(value);
if (std::string(key) == "steps")
- params.sample_params.sample_steps = static_cast<int>(value);
+ if (!params.sample_params.sample_steps)
+ params.sample_params.sample_steps = static_cast<int>(value);
if (field == "shifted_timestep")
- params.sample_params.shifted_timestep = static_cast<int>(value);
+ if (!params.sample_params.shifted_timestep)
+ params.sample_params.shifted_timestep = static_cast<int>(value);
if (field == "video_frames")
- params.video_frames = static_cast<int>(value);
+ if (!params.video_frames)
+ params.video_frames = static_cast<int>(value);
if (field == "fps")
- params.fps = static_cast<int>(value);
+ if (!params.fps)
+ params.fps = static_cast<int>(value);
if (field == "high_noise_steps")
- params.high_noise_sample_params.sample_steps = static_cast<int>(value);
+ if (!params.high_noise_sample_params.sample_steps)
+ params.high_noise_sample_params.sample_steps = static_cast<int>(value);
}
for (const char* key : {"cfg_scale", "strength", "moe_boundary", "img_cfg_scale", "distilled_guidance", "flow_shift", "high_noise_cfg_scale", "high_noise_flow_shift"}) {
if (!defaults.contains(key)) {
@@ -219,21 +226,29 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para
throw std::runtime_error(std::string(key) + " is out of range");
}
if (std::string(key) == "strength")
- params.strength = value;
+ if (!params.strength)
+ params.strength = value;
if (std::string(key) == "moe_boundary")
- params.moe_boundary = value;
+ if (!params.moe_boundary)
+ params.moe_boundary = value;
if (std::string(key) == "cfg_scale")
- params.sample_params.guidance.txt_cfg = value;
+ if (!params.sample_params.guidance.txt_cfg)
+ params.sample_params.guidance.txt_cfg = value;
if (std::string(key) == "img_cfg_scale")
- params.sample_params.guidance.img_cfg = value;
+ if (!params.sample_params.guidance.img_cfg)
+ params.sample_params.guidance.img_cfg = value;
if (std::string(key) == "distilled_guidance")
- params.sample_params.guidance.distilled_guidance = value;
+ if (!params.sample_params.guidance.distilled_guidance)
+ params.sample_params.guidance.distilled_guidance = value;
if (std::string(key) == "flow_shift")
- params.sample_params.flow_shift = value;
+ if (!params.sample_params.flow_shift && !params.high_noise_sample_params.flow_shift)
+ params.sample_params.flow_shift = value;
if (std::string(key) == "high_noise_cfg_scale")
- params.high_noise_sample_params.guidance.txt_cfg = value;
+ if (!params.high_noise_sample_params.guidance.txt_cfg)
+ params.high_noise_sample_params.guidance.txt_cfg = value;
if (std::string(key) == "high_noise_flow_shift")
- params.high_noise_sample_params.flow_shift = value;
+ if (!params.sample_params.flow_shift && !params.high_noise_sample_params.flow_shift)
+ params.high_noise_sample_params.flow_shift = value;
}
if (defaults.contains("custom_sigmas")) {
const auto& sigmas = defaults.at("custom_sigmas");
@@ -243,40 +258,28 @@ static void apply_recipe_defaults(const json& defaults, SDGenerationParams& para
if (!sigma.is_number() || !std::isfinite(sigma.get<float>()) || sigma.get<float>() < 0)
throw std::runtime_error("custom_sigmas must contain finite nonnegative numbers");
}
- params.custom_sigmas = sigmas.get<std::vector<float>>();
+ if (!params.custom_sigmas)
+ params.custom_sigmas = sigmas.get<std::vector<float>>();
}
for (const char* key : {"sampling_method", "high_noise_sampling_method"}) {
if (!defaults.contains(key))
continue;
auto& sample = std::string(key) == "sampling_method" ? params.sample_params : params.high_noise_sample_params;
const auto name = defaults.at(key).get<std::string>();
- bool found = false;
- for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) {
- if (name == sd_sample_method_name(static_cast<sample_method_t>(i))) {
- sample.sample_method = static_cast<sample_method_t>(i);
- found = true;
- break;
- }
- }
- if (!found)
+ if (!example_options::contains(example_options::sample_methods, name))
throw std::runtime_error(std::string("unknown ") + key);
+ if (!sample.sample_method)
+ sample.sample_method = name;
}
for (const char* key : {"scheduler", "high_noise_scheduler"}) {
if (!defaults.contains(key))
continue;
auto& sample = std::string(key) == "scheduler" ? params.sample_params : params.high_noise_sample_params;
const auto name = defaults.at(key).get<std::string>();
- bool found = false;
- for (int i = 0; i < SCHEDULER_COUNT; ++i) {
- if (name == sd_scheduler_name(static_cast<scheduler_t>(i)) ||
- (name == "normal" && i == DISCRETE_SCHEDULER)) {
- sample.scheduler = static_cast<scheduler_t>(i);
- found = true;
- break;
- }
- }
- if (!found)
+ if (!example_options::contains(example_options::schedulers, name) && name != "normal")
throw std::runtime_error(std::string("unknown ") + key);
+ if (!params.sample_params.scheduler && !params.high_noise_sample_params.scheduler)
+ sample.scheduler = name;
}
}
@@ -286,16 +289,20 @@ static void apply_recipe_context(const json& settings, SDContextParams& params)
const auto format = settings.at("vae_format").get<std::string>();
if (format != "auto" && format != "flux" && format != "sd3" && format != "flux2" && format != "wan")
throw std::runtime_error("unknown vae_format");
- params.vae_format = format;
+ if (!params.vae_format)
+ params.vae_format = format;
}
if (settings.contains("model_args")) {
- params.model_args = settings.at("model_args").get<std::string>();
+ if (!params.model_args)
+ params.model_args = settings.at("model_args").get<std::string>();
}
if (settings.contains("backend")) {
- params.backend = settings.at("backend").get<std::string>();
+ if (!params.backend)
+ params.backend = settings.at("backend").get<std::string>();
}
if (settings.contains("params_backend")) {
- params.params_backend = settings.at("params_backend").get<std::string>();
+ if (!params.params_backend)
+ params.params_backend = settings.at("params_backend").get<std::string>();
}
}
diff --git a/examples/server/routes_models.cpp b/examples/server/routes_models.cpp
index f330577..9dbd6ed 100644
--- a/examples/server/routes_models.cpp
+++ b/examples/server/routes_models.cpp
@@ -112,8 +112,8 @@ static void change_model(ServerRuntime& runtime, const httplib::Request& req, ht
*runtime.ctx_params = unload ? models.base_context : std::move(context);
*runtime.default_gen_params = unload ? models.base_defaults : std::move(defaults);
if (!unload) {
- auto params = runtime.ctx_params->to_sd_ctx_params_t();
- models.context.reset(new_sd_ctx(¶ms));
+ auto params = runtime.ctx_params->to_context_opt();
+ models.context.reset(create_sd_context(params));
if (!models.context) {
throw std::runtime_error("Model initialization failed; check the server log for component compatibility and memory errors");
}
diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp
index 30ca7f3..e67bd20 100644
--- a/examples/server/routes_openai.cpp
+++ b/examples/server/routes_openai.cpp
@@ -35,8 +35,8 @@ static bool build_openai_generation_request(const httplib::Request& req,
std::string size = j.value("size", "");
std::string output_format = j.value("output_format", "png");
int output_compression = j.value("output_compression", 100);
- int width = runtime.default_gen_params->width > 0 ? runtime.default_gen_params->width : 512;
- int height = runtime.default_gen_params->height > 0 ? runtime.default_gen_params->height : 512;
+ auto width = runtime.default_gen_params->width;
+ auto height = runtime.default_gen_params->height;
if (!size.empty()) {
auto pos = size.find('x');
if (pos != std::string::npos) {
@@ -120,8 +120,8 @@ static bool build_openai_edit_request(const httplib::Request& req,
}
std::string size = req.form.get_field("size");
- int width = -1;
- int height = -1;
+ auto width = runtime.default_gen_params->width;
+ auto height = runtime.default_gen_params->height;
if (!size.empty()) {
auto pos = size.find('x');
if (pos != std::string::npos) {
@@ -185,12 +185,12 @@ static bool build_openai_edit_request(const httplib::Request& req,
mask_w, mask_h, mask_channel,
0, 0, 1);
request.gen_params.mask_image.reset({(uint32_t)mask_w, (uint32_t)mask_h, 1, mask_raw});
- const sd_image_t& mask_image = request.gen_params.mask_image.get();
+ const SDImage& mask_image = request.gen_params.mask_image.get();
request.gen_params.set_width_and_height_if_unset(mask_image.width, mask_image.height);
} else {
request.gen_params.mask_image.reset({
- (uint32_t)request.gen_params.get_resolved_width(),
- (uint32_t)request.gen_params.get_resolved_height(),
+ (uint32_t)request.gen_params.get_requested_width(),
+ (uint32_t)request.gen_params.get_requested_height(),
1,
nullptr,
});
@@ -218,17 +218,17 @@ static bool execute_sync_img_gen_request(ServerRuntime& runtime,
ImgGenJobRequest& request,
SDImageVec& results,
std::string& error_message) {
- sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
- int num_results = 0;
+ SDOptPtr img_gen_params = request.to_image_opt();
+ int num_results = 0;
{
std::lock_guard<std::mutex> lock(*runtime.sd_ctx_mutex);
PromptImageCallbackGuard prompt_images(runtime.sd_ctx, request.gen_params);
- sd_image_generation_result_t* generated = nullptr;
- generate_image(runtime.sd_ctx, &img_gen_params, &generated);
+ sd_result_t* generated = nullptr;
+ generate_sd_image(runtime.sd_ctx, img_gen_params, &generated);
results.adopt(generated, runtime.ctx_params->lora_model_dir);
num_results = results.count();
- if (!results.empty() && !apply_server_adetailer(runtime, request, img_gen_params, results)) {
+ if (!results.empty() && !apply_server_adetailer(runtime, request, results)) {
error_message = "ADetailer failed";
return false;
}
diff --git a/examples/server/routes_sdapi.cpp b/examples/server/routes_sdapi.cpp
index 26ed7c9..e9f60c7 100644
--- a/examples/server/routes_sdapi.cpp
+++ b/examples/server/routes_sdapi.cpp
@@ -47,41 +47,38 @@ static std::string lower_ascii(std::string value) {
return value;
}
-static enum sample_method_t get_sdapi_sample_method(std::string name) {
- enum sample_method_t result = str_to_sample_method(name.c_str());
- if (result != SAMPLE_METHOD_COUNT) {
- return result;
- }
-
+static std::string get_sdapi_sample_method(std::string name) {
+ if (example_options::contains(example_options::sample_methods, name))
+ return name;
name = lower_ascii(name);
- static const std::unordered_map<std::string_view, sample_method_t> hardcoded{
- {"euler a", EULER_A_SAMPLE_METHOD},
- {"k_euler_a", EULER_A_SAMPLE_METHOD},
- {"euler", EULER_SAMPLE_METHOD},
- {"k_euler", EULER_SAMPLE_METHOD},
- {"heun", HEUN_SAMPLE_METHOD},
- {"k_heun", HEUN_SAMPLE_METHOD},
- {"dpm2", DPM2_SAMPLE_METHOD},
- {"k_dpm_2", DPM2_SAMPLE_METHOD},
- {"lcm", LCM_SAMPLE_METHOD},
- {"ddim", DDIM_TRAILING_SAMPLE_METHOD},
- {"dpm++ 2m", DPMPP2M_SAMPLE_METHOD},
- {"k_dpmpp_2m", DPMPP2M_SAMPLE_METHOD},
- {"dpm++ 2m sde", DPMPP2M_SDE_SAMPLE_METHOD},
- {"k_dpmpp_2m_sde", DPMPP2M_SDE_SAMPLE_METHOD},
- {"dpm++ 2m sde gpu", DPMPP2M_SDE_BT_SAMPLE_METHOD},
- {"k_dpmpp_2m_sde_gpu", DPMPP2M_SDE_BT_SAMPLE_METHOD},
- {"res multistep", RES_MULTISTEP_SAMPLE_METHOD},
- {"k_res_multistep", RES_MULTISTEP_SAMPLE_METHOD},
- {"res 2s", RES_2S_SAMPLE_METHOD},
- {"k_res_2s", RES_2S_SAMPLE_METHOD},
- {"euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
- {"k_euler_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
- {"euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
- {"k_euler_a_cfg_pp", EULER_CFG_PP_SAMPLE_METHOD},
+ static const std::unordered_map<std::string_view, std::string> hardcoded{
+ {"euler a", "euler_a"},
+ {"k_euler_a", "euler_a"},
+ {"euler", "euler"},
+ {"k_euler", "euler"},
+ {"heun", "heun"},
+ {"k_heun", "heun"},
+ {"dpm2", "dpm2"},
+ {"k_dpm_2", "dpm2"},
+ {"lcm", "lcm"},
+ {"ddim", "ddim_trailing"},
+ {"dpm++ 2m", "dpm++2m"},
+ {"k_dpmpp_2m", "dpm++2m"},
+ {"dpm++ 2m sde", "dpm++2m_sde"},
+ {"k_dpmpp_2m_sde", "dpm++2m_sde"},
+ {"dpm++ 2m sde gpu", "dpm++2m_sde_bt"},
+ {"k_dpmpp_2m_sde_gpu", "dpm++2m_sde_bt"},
+ {"res multistep", "res_multistep"},
+ {"k_res_multistep", "res_multistep"},
+ {"res 2s", "res_2s"},
+ {"k_res_2s", "res_2s"},
+ {"euler_cfg_pp", "euler_cfg_pp"},
+ {"k_euler_cfg_pp", "euler_cfg_pp"},
+ {"euler_a_cfg_pp", "euler_a_cfg_pp"},
+ {"k_euler_a_cfg_pp", "euler_a_cfg_pp"},
};
auto it = hardcoded.find(name);
- return it != hardcoded.end() ? it->second : SAMPLE_METHOD_COUNT;
+ return it != hardcoded.end() ? it->second : std::string{};
}
static bool parse_sdapi_adetailer(const json& body,
@@ -191,10 +188,11 @@ static bool parse_sdapi_adetailer(const json& body,
ServerADetailerPass pass;
pass.prompt = text("ad_prompt", "");
pass.negative_prompt = text("ad_negative_prompt", "");
- pass.width = request.gen_params.get_resolved_width();
- pass.height = request.gen_params.get_resolved_height();
- pass.strength = static_cast<float>(number("ad_denoising_strength", 0.4, 0, 1));
- auto append = [&](const char* key, const std::string& value) {
+ pass.width = request.gen_params.width;
+ pass.height = request.gen_params.height;
+ if (config.contains("ad_denoising_strength"))
+ pass.strength = static_cast<float>(number("ad_denoising_strength", 0, 0, 1));
+ auto append = [&](const char* key, const std::string& value) {
if (!pass.extra_args.empty())
pass.extra_args += ",";
pass.extra_args += std::string(key) + "=" + value;
@@ -218,7 +216,7 @@ static bool parse_sdapi_adetailer(const json& body,
append("mask_min_ratio", json(min_ratio).dump());
append("mask_max_ratio", json(max_ratio).dump());
// Bound pixel operations before converting them to the native mask parameters.
- const int mask_limit = std::clamp(std::max(pass.width, pass.height), 32, 16384);
+ const int mask_limit = k_max_generation_dimension;
append_integer("dilate_erode", "ad_dilate_erode", 4, -mask_limit, mask_limit);
append_integer("x_offset", "ad_x_offset", 0, -mask_limit, mask_limit);
append_integer("y_offset", "ad_y_offset", 0, -mask_limit, mask_limit);
@@ -231,8 +229,12 @@ static bool parse_sdapi_adetailer(const json& body,
append("mask_mode", merge == "None" ? "none" : merge == "Merge" ? "merge"
: "merge_invert");
if (boolean("ad_use_inpaint_width_height", false)) {
- pass.width = integer("ad_inpaint_width", 512, 1, k_max_generation_dimension);
- pass.height = integer("ad_inpaint_height", 512, 1, k_max_generation_dimension);
+ pass.width.reset();
+ pass.height.reset();
+ if (config.contains("ad_inpaint_width"))
+ pass.width = integer("ad_inpaint_width", 0, 1, k_max_generation_dimension);
+ if (config.contains("ad_inpaint_height"))
+ pass.height = integer("ad_inpaint_height", 0, 1, k_max_generation_dimension);
}
if (boolean("ad_use_steps", false))
append_integer("steps", "ad_steps", 28, 1, k_max_adetailer_steps);
@@ -251,20 +253,20 @@ static bool parse_sdapi_adetailer(const json& body,
if (legacy_karras)
sampler.resize(sampler.size() - suffix.size());
const auto method = get_sdapi_sample_method(sampler);
- if (method == SAMPLE_METHOD_COUNT)
+ if (method.empty())
throw std::invalid_argument("unsupported ADetailer sampler");
- append("sample_method", sd_sample_method_name(method));
+ append("sample_method", method);
if (legacy_karras && text("ad_scheduler", "Use same scheduler") == "Use same scheduler")
- append("scheduler", sd_scheduler_name(KARRAS_SCHEDULER));
+ append("scheduler", "karras");
}
const auto scheduler = text("ad_scheduler", "Use same scheduler");
if (scheduler == "Automatic") {
pass.automatic_scheduler = true;
} else if (scheduler != "Use same scheduler") {
- const auto value = str_to_scheduler(scheduler.c_str());
- if (value == SCHEDULER_COUNT)
+ const auto value = scheduler;
+ if (!example_options::contains(example_options::schedulers, value) && value != "normal")
throw std::invalid_argument("unsupported ADetailer scheduler");
- append("scheduler", sd_scheduler_name(value));
+ append("scheduler", value == "normal" ? "discrete" : value);
}
}
request.adetailer_passes.push_back(std::move(pass));
@@ -287,21 +289,9 @@ static bool build_sdapi_img_gen_request(const json& j,
std::string& error_message) {
std::string prompt = j.value("prompt", "");
std::string negative_prompt = j.value("negative_prompt", "");
- int width = j.value("width", 512);
- int height = j.value("height", 512);
- int steps = j.value("steps", runtime.default_gen_params->sample_params.sample_steps);
- float cfg_scale = j.value("cfg_scale", runtime.default_gen_params->sample_params.guidance.txt_cfg);
- int64_t seed = j.value("seed", -1);
- int batch_size = j.value("batch_size", 1);
- int clip_skip = j.value("clip_skip", -1);
std::string sampler_name = j.value("sampler_name", "");
std::string scheduler_name = j.value("scheduler", "");
- if (width <= 0 || height <= 0) {
- error_message = "width and height must be positive";
- return false;
- }
-
if (prompt.empty()) {
error_message = "prompt required";
return false;
@@ -309,15 +299,20 @@ static bool build_sdapi_img_gen_request(const json& j,
request.gen_params = *runtime.default_gen_params;
- request.gen_params.prompt = prompt;
- request.gen_params.negative_prompt = negative_prompt;
- request.gen_params.seed = seed;
- request.gen_params.sample_params.sample_steps = steps;
- request.gen_params.batch_count = batch_size;
- request.gen_params.sample_params.guidance.txt_cfg = cfg_scale;
- const bool recipe_dimensions = !img2img && model_status(runtime)["recipe"].is_string();
- request.gen_params.width = j.value("width", recipe_dimensions ? runtime.default_gen_params->width : -1);
- request.gen_params.height = j.value("height", recipe_dimensions ? runtime.default_gen_params->height : -1);
+ request.gen_params.prompt = prompt;
+ if (j.contains("negative_prompt"))
+ request.gen_params.negative_prompt = negative_prompt;
+ auto load_input = [&](const char* key, auto& value) {
+ if (j.contains(key))
+ j.at(key).get_to(value);
+ };
+ load_input("seed", request.gen_params.seed);
+ load_input("steps", request.gen_params.sample_params.sample_steps);
+ load_input("batch_size", request.gen_params.batch_count);
+ load_input("cfg_scale", request.gen_params.sample_params.guidance.txt_cfg);
+ load_input("width", request.gen_params.width);
+ load_input("height", request.gen_params.height);
+ load_input("clip_skip", request.gen_params.clip_skip);
if (!img2img && j.value("enable_hr", false)) {
request.gen_params.hires_enabled = true;
@@ -337,17 +332,13 @@ static bool build_sdapi_img_gen_request(const json& j,
return false;
}
- if (clip_skip > 0) {
- request.gen_params.clip_skip = clip_skip;
- }
-
- enum sample_method_t sample_method = get_sdapi_sample_method(sampler_name);
- if (sample_method != SAMPLE_METHOD_COUNT) {
+ const auto sample_method = get_sdapi_sample_method(sampler_name);
+ if (!sample_method.empty()) {
request.gen_params.sample_params.sample_method = sample_method;
}
- enum scheduler_t scheduler = str_to_scheduler(scheduler_name.c_str());
- if (scheduler != SCHEDULER_COUNT) {
+ const auto scheduler = scheduler_name;
+ if (example_options::contains(example_options::schedulers, scheduler) || scheduler == "normal") {
request.gen_params.sample_params.scheduler = scheduler;
}
@@ -404,7 +395,7 @@ static bool build_sdapi_img_gen_request(const json& j,
0,
0,
request.gen_params.init_image)) {
- const sd_image_t& image = request.gen_params.init_image.get();
+ const SDImage& image = request.gen_params.init_image.get();
request.gen_params.set_width_and_height_if_unset(image.width, image.height);
}
}
@@ -415,10 +406,10 @@ static bool build_sdapi_img_gen_request(const json& j,
0,
0,
request.gen_params.mask_image)) {
- const sd_image_t& image = request.gen_params.mask_image.get();
+ const SDImage& image = request.gen_params.mask_image.get();
request.gen_params.set_width_and_height_if_unset(image.width, image.height);
}
- sd_image_t& mask_image = request.gen_params.mask_image.get();
+ SDImage& mask_image = request.gen_params.mask_image.get();
bool inpainting_mask_invert = j.value("inpainting_mask_invert", 0) != 0;
if (inpainting_mask_invert && mask_image.data != nullptr) {
for (uint32_t i = 0; i < mask_image.width * mask_image.height; ++i) {
@@ -445,7 +436,7 @@ static bool build_sdapi_img_gen_request(const json& j,
0,
0, 0,
image_owner)) {
- const sd_image_t& image = image_owner.get();
+ const SDImage& image = image_owner.get();
request.gen_params.set_width_and_height_if_unset(image.width, image.height);
request.gen_params.ref_images.push_back(std::move(image_owner));
}
@@ -499,14 +490,18 @@ struct SDAPIPreviewState {
static SDAPIPreviewState g_preview;
-static void preview_callback(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data) {
+static void preview_callback(int step, int frame_count, const sd_tensor_t* const* tensors, bool is_noisy, void* data) {
+ std::vector<SDImage> views;
+ for (int i = 0; i < frame_count; ++i)
+ views.push_back(tensor_image_view(tensors[i]));
+ SDImage* frames = views.data();
(void)is_noisy;
(void)data;
if (frame_count <= 0 || frames == nullptr || frames[0].data == nullptr) {
return;
}
- const sd_image_t& frame = frames[0];
+ const SDImage& frame = frames[0];
const sd_preview_info_t info = sd_get_preview_info();
auto image_bytes = encode_image_to_vector(EncodedImageFormat::JPEG,
@@ -530,7 +525,7 @@ static void preview_callback(int step, int frame_count, sd_image_t* frames, bool
}
struct SDAPIPreviewGuard {
- SDAPIPreviewGuard(const std::string& id_task, const sd_img_gen_params_t& params, const std::string& job_type, bool detailing = false) {
+ SDAPIPreviewGuard(const std::string& id_task, const SDGenerationParams& params, const std::string& job_type, bool detailing = false) {
std::lock_guard<std::mutex> lock(g_preview.mutex);
g_preview.id_task = id_task;
g_preview.total = 0;
@@ -539,13 +534,13 @@ struct SDAPIPreviewGuard {
g_preview.step = 0;
g_preview.active = true;
// Each batch item has a base pass and, when enabled, a hires pass.
- g_preview.job_count = static_cast<int64_t>(std::max(1, params.batch_count)) * (params.hires.enabled ? 2 : 1) + (detailing ? 1 : 0);
+ g_preview.job_count = static_cast<int64_t>(std::max(1, params.batch_count.value_or(1))) * (params.hires_enabled.value_or(false) ? 2 : 1) + (detailing ? 1 : 0);
g_preview.job_no = 0;
- sd_set_preview_callback(preview_callback, PREVIEW_PROJ, 1, true, false, nullptr);
+ sd_set_preview_callback(preview_callback, "proj", 1, true, false, nullptr);
}
void begin_detailing() {
- sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr);
+ sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr);
std::lock_guard<std::mutex> lock(g_preview.mutex);
g_preview.job_type = "adetailer";
g_preview.job_no = static_cast<int>(g_preview.job_count - 1);
@@ -554,7 +549,7 @@ struct SDAPIPreviewGuard {
}
~SDAPIPreviewGuard() {
- sd_set_preview_callback(nullptr, PREVIEW_NONE, 1, false, false, nullptr);
+ sd_set_preview_callback(nullptr, "none", 1, false, false, nullptr);
std::lock_guard<std::mutex> lock(g_preview.mutex);
g_preview.active = false;
g_preview.b64.clear();
@@ -594,7 +589,7 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
LOG_DEBUG("%s\n", request.gen_params.to_string().c_str());
- sd_img_gen_params_t img_gen_params = request.to_sd_img_gen_params_t();
+ SDOptPtr img_gen_params = request.to_image_opt();
SDImageVec results;
int num_results = 0;
@@ -602,14 +597,14 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
std::lock_guard<std::mutex> lock(*runtime->sd_ctx_mutex);
PromptImageCallbackGuard prompt_images(runtime->sd_ctx, request.gen_params);
const bool detailing = request.adetailer_enabled && !runtime->default_gen_params->ad_model_path.empty();
- SDAPIPreviewGuard preview_guard(id_task, img_gen_params, img2img ? "img2img" : "txt2img", detailing);
- sd_image_generation_result_t* generated = nullptr;
- generate_image(runtime->sd_ctx, &img_gen_params, &generated);
+ SDAPIPreviewGuard preview_guard(id_task, request.gen_params, img2img ? "img2img" : "txt2img", detailing);
+ sd_result_t* generated = nullptr;
+ generate_sd_image(runtime->sd_ctx, img_gen_params, &generated);
results.adopt(generated, runtime->ctx_params->lora_model_dir);
num_results = results.count();
if (detailing && !results.empty())
preview_guard.begin_detailing();
- if (!results.empty() && !apply_server_adetailer(*runtime, request, img_gen_params, results)) {
+ if (!results.empty() && !apply_server_adetailer(*runtime, request, results)) {
res.status = 500;
res.set_content(R"({"error":"ADetailer failed"})", "application/json");
return;
@@ -748,8 +743,8 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
routes.get("/sdapi/v1/samplers", [runtime](const httplib::Request&, httplib::Response& res) {
std::vector<std::string> sampler_names;
sampler_names.push_back("default");
- for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) {
- sampler_names.push_back(sd_sample_method_name((sample_method_t)i));
+ for (const char* name : example_options::sample_methods) {
+ sampler_names.push_back(name);
}
json r = json::array();
for (auto name : sampler_names) {
@@ -765,9 +760,9 @@ void register_sdapi_endpoints(httplib::Server& svr, ServerRuntime& rt) {
routes.get("/sdapi/v1/schedulers", [runtime](const httplib::Request&, httplib::Response& res) {
std::vector<std::string> scheduler_names;
scheduler_names.push_back("default");
- for (int i = 0; i < SCHEDULER_COUNT; i++) {
- scheduler_names.push_back(sd_scheduler_name((scheduler_t)i));
- if (i == DISCRETE_SCHEDULER) {
+ for (const std::string name : example_options::schedulers) {
+ scheduler_names.push_back(name);
+ if (name == "discrete") {
scheduler_names.push_back("normal");
}
}
diff --git a/examples/server/routes_sdcpp.cpp b/examples/server/routes_sdcpp.cpp
index 1d11b89..c6e4971 100644
--- a/examples/server/routes_sdcpp.cpp
+++ b/examples/server/routes_sdcpp.cpp
@@ -15,7 +15,7 @@ namespace fs = std::filesystem;
static constexpr uint32_t k_max_upscale_dimension = 8192;
-static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int repeats) {
+static bool valid_upscale_dimensions(const SDImage& image, int factor, int repeats) {
if (image.width == 0 || image.height == 0 || factor < 1 || repeats < 1 || repeats > 4) {
return false;
}
@@ -31,51 +31,20 @@ static bool valid_upscale_dimensions(const sd_image_t& image, int factor, int re
return true;
}
-static bool parse_cache_mode(const std::string& mode_str, sd_cache_mode_t& mode_out) {
- if (mode_str == "disabled") {
- mode_out = SD_CACHE_DISABLED;
- return true;
- }
- if (mode_str == "easycache") {
- mode_out = SD_CACHE_EASYCACHE;
- return true;
- }
- if (mode_str == "ucache") {
- mode_out = SD_CACHE_UCACHE;
- return true;
- }
- if (mode_str == "dbcache") {
- mode_out = SD_CACHE_DBCACHE;
- return true;
- }
- if (mode_str == "taylorseer") {
- mode_out = SD_CACHE_TAYLORSEER;
- return true;
- }
- if (mode_str == "cache-dit") {
- mode_out = SD_CACHE_CACHE_DIT;
- return true;
- }
- if (mode_str == "spectrum") {
- mode_out = SD_CACHE_SPECTRUM;
- return true;
- }
- return false;
-}
-
-static json finite_number_or_null(float value) {
- return std::isfinite(value) ? json(value) : json(nullptr);
-}
-
-static const char* capability_scheduler_name(enum scheduler_t scheduler) {
- return scheduler < SCHEDULER_COUNT ? sd_scheduler_name(scheduler) : "default";
+static bool parse_cache_mode(const std::string& value, std::string& mode) {
+ const std::string canonical = value == "cache-dit" ? "cache_dit" : value;
+ SDOptPtr opt(sd_img_gen_opt_create());
+ if (!opt || sd_opt_set_string(opt.get(), "cache.mode", canonical.c_str()) != SD_STATUS_OK)
+ return false;
+ mode = canonical;
+ return true;
}
-static const char* capability_sample_method_name(enum sample_method_t sample_method) {
- return sample_method < SAMPLE_METHOD_COUNT ? sd_sample_method_name(sample_method) : "default";
+static json finite_number_or_null(const std::optional<float>& value) {
+ return value && std::isfinite(*value) ? json(*value) : json(nullptr);
}
-static json make_vae_tiling_json(const sd_tiling_params_t& params) {
+static json make_vae_tiling_json(const SDTilingParams& params) {
return {
{"enabled", params.enabled},
{"temporal_tiling", params.temporal_tiling},
@@ -99,11 +68,11 @@ static fs::path resolve_display_model_path(const ServerRuntime& runtime) {
return {};
}
-static json make_sample_params_json(const sd_sample_params_t& sample_params, const std::vector<int>& skip_layers, const std::vector<float>& custom_sigmas = {}) {
+static json make_sample_params_json(const SDSampleParams& sample_params, const std::optional<std::vector<int>>& skip_layers, const std::optional<std::vector<float>>& custom_sigmas = {}) {
const auto& guidance = sample_params.guidance;
return {
- {"scheduler", capability_scheduler_name(sample_params.scheduler)},
- {"sample_method", capability_sample_method_name(sample_params.sample_method)},
+ {"scheduler", sample_params.scheduler},
+ {"sample_method", sample_params.sample_method},
{"sample_steps", sample_params.sample_steps},
{"custom_sigmas", custom_sigmas},
{"eta", finite_number_or_null(sample_params.eta)},
@@ -144,8 +113,8 @@ static json make_img_gen_defaults_json(const SDGenerationParams& defaults, const
{"prompt", defaults.prompt},
{"negative_prompt", defaults.negative_prompt},
{"clip_skip", defaults.clip_skip},
- {"width", defaults.width > 0 ? defaults.width : 512},
- {"height", defaults.height > 0 ? defaults.height : 512},
+ {"width", defaults.width},
+ {"height", defaults.height},
{"strength", defaults.strength},
{"seed", defaults.seed},
{"batch_count", defaults.batch_count},
@@ -173,8 +142,8 @@ static json make_vid_gen_defaults_json(const SDGenerationParams& defaults, const
{"prompt", defaults.prompt},
{"negative_prompt", defaults.negative_prompt},
{"clip_skip", defaults.clip_skip},
- {"width", defaults.width > 0 ? defaults.width : 512},
- {"height", defaults.height > 0 ? defaults.height : 512},
+ {"width", defaults.width},
+ {"height", defaults.height},
{"strength", defaults.strength},
{"seed", defaults.seed},
{"video_frames", defaults.video_frames},
@@ -234,8 +203,8 @@ static json make_vid_gen_features_json() {
static json make_preview_modes_json() {
json modes = json::array();
- for (int i = 0; i < PREVIEW_COUNT; ++i) {
- modes.push_back(sd_preview_name((preview_t)i));
+ for (const char* name : example_options::preview_modes) {
+ modes.push_back(name);
}
return modes;
}
@@ -258,13 +227,13 @@ static json make_capabilities_json(ServerRuntime& runtime) {
json available_upscalers = json::array();
json supported_modes = json::array();
- for (int i = 0; i < SAMPLE_METHOD_COUNT; ++i) {
- samplers.push_back(sd_sample_method_name((sample_method_t)i));
+ for (const char* name : example_options::sample_methods) {
+ samplers.push_back(name);
}
- for (int i = 0; i < SCHEDULER_COUNT; ++i) {
- schedulers.push_back(sd_scheduler_name((scheduler_t)i));
- if (i == DISCRETE_SCHEDULER) {
+ for (const std::string name : example_options::schedulers) {
+ schedulers.push_back(name);
+ if (name == "discrete") {
schedulers.push_back("normal");
}
}
@@ -483,6 +452,21 @@ static bool parse_vid_gen_request(const json& body,
return false;
}
+ auto& params = request.gen_params;
+ auto merge_shared_sample_input = [&](const char* key, auto member) {
+ const auto base = body.find("sample_params");
+ const auto high = body.find("high_noise_sample_params");
+ if (base != body.end() && base->is_object() && base->contains(key)) {
+ if (base->at(key).is_null())
+ (params.high_noise_sample_params.*member).reset();
+ } else if (high != body.end() && high->is_object() && high->contains(key)) {
+ params.sample_params.*member = params.high_noise_sample_params.*member;
+ }
+ };
+ merge_shared_sample_input("scheduler", &SDSampleParams::scheduler);
+ merge_shared_sample_input("flow_shift", &SDSampleParams::flow_shift);
+ merge_shared_sample_input("shifted_timestep", &SDSampleParams::shifted_timestep);
+
std::string output_format = body.value("output_format", "webm");
int output_compression = body.value("output_compression", 100);
if (!assign_output_options(request, output_format, output_compression, error_message)) {
@@ -546,7 +530,12 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
res.set_content(json({{"error", error_message}}).dump(), "application/json");
return;
}
- const int tile_size = std::max(32, body.value("tile_size", runtime->default_gen_params->upscale_tile_size));
+ const auto tile_size = body.value("tile_size", runtime->default_gen_params->upscale_tile_size);
+ if (tile_size && *tile_size < 32) {
+ res.status = 400;
+ res.set_content(json({{"error", "tile_size must be at least 32"}}).dump(), "application/json");
+ return;
+ }
const int repeats = std::clamp(body.value("repeats", 1), 1, 4);
const std::string wanted = body.value("upscaler", std::string());
@@ -597,18 +586,18 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
// One GPU: an upscale must not run while a generation is using it.
std::lock_guard<std::mutex> ctx_lock(*runtime->sd_ctx_mutex);
- UpscalerCtxPtr upscaler_ctx(new_upscaler_ctx(model_path.c_str(),
- runtime->ctx_params->diffusion_conv_direct,
- runtime->ctx_params->n_threads,
- tile_size,
- runtime->ctx_params->backend.c_str(),
- runtime->ctx_params->params_backend.c_str()));
+ UpscalerCtxPtr upscaler_ctx(create_sd_upscaler(model_path.c_str(),
+ runtime->ctx_params->diffusion_conv_direct,
+ runtime->ctx_params->n_threads,
+ tile_size,
+ runtime->ctx_params->backend ? runtime->ctx_params->backend->c_str() : nullptr,
+ runtime->ctx_params->params_backend ? runtime->ctx_params->params_backend->c_str() : nullptr));
if (upscaler_ctx == nullptr) {
res.status = 500;
res.set_content(R"({"error":"the upscaler model could not be loaded"})", "application/json");
return;
}
- const int factor = get_upscale_factor(upscaler_ctx.get());
+ const int factor = sd_upscaler_get_scale(upscaler_ctx.get());
// The model file may have changed since its metadata was cached.
if (!valid_upscale_dimensions(input.get(), factor, repeats)) {
res.status = 400;
@@ -619,24 +608,24 @@ void register_sdcpp_api_endpoints(httplib::Server& svr, ServerRuntime& rt) {
SDImageOwner current(input.release());
std::string generation_info;
for (int i = 0; i < repeats; ++i) {
- sd_image_generation_result_t* generated = nullptr;
- if (!upscale(upscaler_ctx.get(), current.get(), (uint32_t)factor, &generated) ||
- !generated || generated->count <= 0 || generated->items[0].image.data == nullptr) {
- free_sd_image_generation_result(generated);
+ sd_result_t* generated = nullptr;
+ if (!upscale_sd_image(upscaler_ctx.get(), current.get(), &generated) ||
+ !generated || sd_result_get_image_count(generated) <= 0 || sd_tensor_data(sd_result_get_image(generated, 0)) == nullptr) {
+ sd_result_free(generated);
res.status = 500;
res.set_content(R"({"error":"upscale failed"})", "application/json");
return;
}
- generation_info = merge_generation_info(generation_info, generated->items[0].generation_info);
- sd_image_t produced = generated->items[0].image;
- generated->items[0].image = {};
- free_sd_image_generation_result(generated);
+ generation_info = merge_generation_info(generation_info, sd_result_get_generation_info(generated, 0));
+ SDImage produced = take_result_image(generated, 0);
+
+ sd_result_free(generated);
current.reset(produced);
}
const std::string& format = output_options.output_format;
const int compression = output_options.output_compression;
- const sd_image_t result = current.get();
+ const SDImage result = current.get();
auto image_bytes = encode_image_to_vector(format == "jpeg" ? EncodedImageFormat::JPEG
: format == "webp" ? EncodedImageFormat::WEBP
: EncodedImageFormat::PNG,
diff --git a/examples/server/runtime.cpp b/examples/server/runtime.cpp
index 499693a..6ac3866 100644
--- a/examples/server/runtime.cpp
+++ b/examples/server/runtime.cpp
@@ -19,15 +19,17 @@
namespace fs = std::filesystem;
-bool PromptImageCallbackGuard::load_image(const char* source, void* data, sd_image_t* image_out) {
- const auto& params = static_cast<PromptImageCallbackGuard*>(data)->params_;
+bool PromptImageCallbackGuard::load_image(const char* source, void* data, const sd_tensor_t** image_out) {
+ auto& guard = *static_cast<PromptImageCallbackGuard*>(data);
+ const auto& params = guard.params_;
const auto it = params.prompt_images.find(source);
if (it == params.prompt_images.end()) {
LOG_ERROR("Unknown prompt image ID: %s", source);
return false;
}
- *image_out = it->second.get();
- return image_out->data != nullptr;
+ guard.image_tensor_ = image_tensor(it->second.get());
+ *image_out = guard.image_tensor_.get();
+ return *image_out != nullptr;
}
PromptImageCallbackGuard::PromptImageCallbackGuard(sd_ctx_t* ctx, const SDGenerationParams& params)
@@ -42,34 +44,35 @@ PromptImageCallbackGuard::~PromptImageCallbackGuard() {
bool validate_server_img_gen_request(const ServerRuntime& runtime,
const ImgGenJobRequest& request,
std::string& error_message) {
- auto dimensions_valid = [&](double width, double height) {
- if (!std::isfinite(width) || !std::isfinite(height) ||
- width < 1 || height < 1 || width > k_max_generation_dimension || height > k_max_generation_dimension) {
- error_message = "generation width and height must be between 1 and " + std::to_string(k_max_generation_dimension);
+ auto within_dimension_limit = [&](double value) {
+ if (!std::isfinite(value) || value > k_max_generation_dimension) {
+ error_message = "generation dimensions must not exceed " + std::to_string(k_max_generation_dimension);
return false;
}
return true;
};
const auto& params = request.gen_params;
- const int width = params.get_resolved_width();
- const int height = params.get_resolved_height();
- if (!dimensions_valid(width, height)) {
+ if ((params.width && !within_dimension_limit(*params.width)) ||
+ (params.height && !within_dimension_limit(*params.height)))
return false;
- }
- if (params.hires_enabled) {
- const double hires_width = params.hires_width > 0 ? params.hires_width : params.hires_height > 0 ? params.hires_height
- : std::round(width * params.hires_scale);
- const double hires_height = params.hires_height > 0 ? params.hires_height : params.hires_width > 0 ? params.hires_width
- : std::round(height * params.hires_scale);
- if (!dimensions_valid(hires_width, hires_height)) {
+ if (params.hires_enabled.value_or(false)) {
+ if ((params.hires_width && !within_dimension_limit(*params.hires_width)) ||
+ (params.hires_height && !within_dimension_limit(*params.hires_height)))
return false;
+ if (params.hires_width.value_or(0) <= 0 && params.hires_height.value_or(0) <= 0) {
+ // Use the documented library defaults for this resource-policy check only.
+ const float scale = params.hires_scale.value_or(2.f);
+ if (!within_dimension_limit(std::round(params.width.value_or(512) * scale)) ||
+ !within_dimension_limit(std::round(params.height.value_or(512) * scale)))
+ return false;
}
}
if (!request.adetailer_enabled || runtime.default_gen_params->ad_model_path.empty()) {
return true;
}
auto pass_valid = [&](const ServerADetailerPass& pass) {
- if (!dimensions_valid(pass.width, pass.height)) {
+ if ((pass.width && !within_dimension_limit(*pass.width)) ||
+ (pass.height && !within_dimension_limit(*pass.height))) {
return false;
}
// Match the native key normalization, but leave full option parsing to ADetailer.
@@ -114,10 +117,9 @@ bool validate_server_img_gen_request(const ServerRuntime& runtime,
}
bool apply_server_adetailer(ServerRuntime& runtime,
- const ImgGenJobRequest& request,
- const sd_img_gen_params_t& params,
+ ImgGenJobRequest& request,
SDImageVec& results) {
- const auto& gen_params = request.gen_params;
+ auto& gen_params = request.gen_params;
const auto& ad_model_path = runtime.default_gen_params->ad_model_path;
if (!request.adetailer_enabled || ad_model_path.empty()) {
return true;
@@ -137,53 +139,54 @@ bool apply_server_adetailer(ServerRuntime& runtime,
passes.push_back(std::move(pass));
}
- ADetailerCtxPtr ad_ctx(new_adetailer_ctx(ad_model_path.c_str(),
- runtime.ctx_params->n_threads,
- runtime.ctx_params->effective_backend.c_str(),
- runtime.ctx_params->effective_params_backend.c_str()));
+ ADetailerCtxPtr ad_ctx(create_sd_adetailer(ad_model_path.c_str(),
+ runtime.ctx_params->n_threads,
+ runtime.ctx_params->effective_backend.c_str(),
+ runtime.ctx_params->effective_params_backend.c_str()));
if (ad_ctx == nullptr) {
- LOG_ERROR("new_adetailer_ctx failed");
+ LOG_ERROR("sd_adetailer_create failed");
return false;
}
for (const auto& pass : passes) {
- sd_adetailer_params_t ad_params{};
- ad_params.prompt = pass.prompt.empty() ? nullptr : pass.prompt.c_str();
- ad_params.negative_prompt = pass.negative_prompt.empty() ? nullptr : pass.negative_prompt.c_str();
- ad_params.extra_ad_args = pass.extra_args.c_str();
for (int i = 0; i < results.count(); ++i) {
if (results[i].data == nullptr) {
continue;
}
- sd_img_gen_params_t ad_generation_params = params;
- ad_generation_params.seed = generation_seed(results.generation_info(i));
- ad_generation_params.width = pass.width;
- ad_generation_params.height = pass.height;
- ad_generation_params.strength = pass.strength;
- if (pass.clip_skip > 0) {
- ad_generation_params.clip_skip = pass.clip_skip;
+ SDOptBuilder ad_options(gen_params.to_image_opt(true).release());
+ ad_options.unset("width");
+ ad_options.unset("height");
+ ad_options.unset("strength");
+ ad_options.set_string("adetail.prompt", pass.prompt);
+ ad_options.set_string("adetail.negative_prompt", pass.negative_prompt);
+ ad_options.set_string("adetail.args", pass.extra_args);
+ ad_options.set_int("seed", generation_seed(results.generation_info(i)));
+ ad_options.set_int("width", pass.width);
+ ad_options.set_int("height", pass.height);
+ ad_options.set_float("strength", pass.strength);
+ if (pass.clip_skip) {
+ ad_options.set_int("clip_skip", pass.clip_skip);
}
if (pass.automatic_scheduler) {
- ad_generation_params.sample_params.scheduler = SCHEDULER_COUNT;
+ ad_options.set_string("sample.scheduler", "auto");
}
- sd_image_generation_result_t* detailed = nullptr;
- if (!adetail_image(ad_ctx.get(),
- runtime.sd_ctx,
- results[i],
- &ad_params,
- &ad_generation_params,
- &detailed) ||
- detailed == nullptr || detailed->count <= 0 || detailed->items[0].image.data == nullptr) {
- free_sd_image_generation_result(detailed);
+ sd_result_t* detailed = nullptr;
+ if (!adetail_sd_image(ad_ctx.get(),
+ runtime.sd_ctx,
+ results[i],
+ ad_options.finish(),
+ &detailed) ||
+ detailed == nullptr || sd_result_get_image_count(detailed) <= 0 || sd_tensor_data(sd_result_get_image(detailed, 0)) == nullptr) {
+ sd_result_free(detailed);
LOG_ERROR("ADetailer failed for image %d", i + 1);
return false;
}
- results.append_generation_info(i, detailed->items[0].generation_info, runtime.ctx_params->lora_model_dir);
- free(results[i].data);
- results[i] = detailed->items[0].image;
- detailed->items[0].image = {};
- free_sd_image_generation_result(detailed);
+ results.append_generation_info(i, sd_result_get_generation_info(detailed, 0), runtime.ctx_params->lora_model_dir);
+ sd_image_free(&results[i]);
+ results[i] = take_result_image(detailed, 0);
+
+ sd_result_free(detailed);
}
}
return true;
@@ -582,7 +585,7 @@ void refresh_upscaler_cache(ServerRuntime& rt) {
});
upscaler_entry.image_upscale_factor = previous != previous_cache.end()
? previous->image_upscale_factor
- : get_upscaler_model_scale(upscaler_entry.fullpath.c_str());
+ : sd_upscaler_probe_scale(upscaler_entry.fullpath.c_str());
if (upscaler_entry.image_upscale_factor > 0) {
upscaler_entry.scale = upscaler_entry.image_upscale_factor;
upscaler_entry.model_name = "ESRGAN_" + std::to_string(upscaler_entry.scale) + "x";
diff --git a/examples/server/runtime.h b/examples/server/runtime.h
index b21f6f5..abb9766 100644
--- a/examples/server/runtime.h
+++ b/examples/server/runtime.h
@@ -73,10 +73,10 @@ struct ServerADetailerPass {
std::string prompt;
std::string negative_prompt;
std::string extra_args;
- int width = 512;
- int height = 512;
- float strength = 0.4f;
- int clip_skip = -1;
+ std::optional<int> width;
+ std::optional<int> height;
+ std::optional<float> strength;
+ std::optional<int> clip_skip;
bool automatic_scheduler = false;
};
@@ -89,16 +89,15 @@ struct ImgGenJobRequest {
bool adetailer_enabled = true;
std::vector<ServerADetailerPass> adetailer_passes;
- sd_img_gen_params_t to_sd_img_gen_params_t() {
- return gen_params.to_sd_img_gen_params_t();
- }
+ SDOptPtr to_image_opt() { return gen_params.to_image_opt(); }
};
class PromptImageCallbackGuard {
sd_ctx_t* ctx_;
const SDGenerationParams& params_;
+ SDTensorPtr image_tensor_;
- static bool load_image(const char* source, void* data, sd_image_t* image_out);
+ static bool load_image(const char* source, void* data, const sd_tensor_t** image_out);
public:
PromptImageCallbackGuard(sd_ctx_t* ctx, const SDGenerationParams& params);
@@ -121,9 +120,7 @@ struct VidGenJobRequest {
std::string preview_mode = "none";
int preview_interval = 1;
- sd_vid_gen_params_t to_sd_vid_gen_params_t() {
- return gen_params.to_sd_vid_gen_params_t();
- }
+ SDOptPtr to_video_opt() { return gen_params.to_video_opt(); }
};
std::string base64_encode(const std::vector<uint8_t>& bytes);
@@ -147,6 +144,5 @@ std::string get_lora_full_path(ServerRuntime& rt, const std::string& path);
void refresh_upscaler_cache(ServerRuntime& rt);
int64_t unix_timestamp_now_ms();
bool apply_server_adetailer(ServerRuntime& runtime,
- const ImgGenJobRequest& request,
- const sd_img_gen_params_t& params,
+ ImgGenJobRequest& request,
SDImageVec& results);
diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h
index 3a06aea..c5a25c6 100644
--- a/include/stable-diffusion.h
+++ b/include/stable-diffusion.h
@@ -19,487 +19,222 @@
#endif
#endif
-#ifdef __cplusplus
-extern "C" {
-#endif
-
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
-enum rng_type_t {
- STD_DEFAULT_RNG,
- CUDA_RNG,
- CPU_RNG,
- RNG_TYPE_COUNT
-};
-
-enum sample_method_t {
- EULER_SAMPLE_METHOD,
- EULER_A_SAMPLE_METHOD,
- HEUN_SAMPLE_METHOD,
- DPM2_SAMPLE_METHOD,
- DPMPP2S_A_SAMPLE_METHOD,
- DPMPP2M_SAMPLE_METHOD,
- DPMPP2Mv2_SAMPLE_METHOD,
- IPNDM_SAMPLE_METHOD,
- IPNDM_V_SAMPLE_METHOD,
- LCM_SAMPLE_METHOD,
- DDIM_TRAILING_SAMPLE_METHOD,
- TCD_SAMPLE_METHOD,
- RES_MULTISTEP_SAMPLE_METHOD,
- RES_2S_SAMPLE_METHOD,
- ER_SDE_SAMPLE_METHOD,
- EULER_CFG_PP_SAMPLE_METHOD,
- EULER_A_CFG_PP_SAMPLE_METHOD,
- EULER_GE_SAMPLE_METHOD,
- DPMPP2M_SDE_SAMPLE_METHOD,
- DPMPP2M_SDE_BT_SAMPLE_METHOD,
- LMS_SAMPLE_METHOD,
- SAMPLE_METHOD_COUNT
-};
-
-extern SD_API const char* sample_method_to_str[];
-
-enum scheduler_t {
- DISCRETE_SCHEDULER,
- KARRAS_SCHEDULER,
- EXPONENTIAL_SCHEDULER,
- AYS_SCHEDULER,
- GITS_SCHEDULER,
- SGM_UNIFORM_SCHEDULER,
- SIMPLE_SCHEDULER,
- SMOOTHSTEP_SCHEDULER,
- KL_OPTIMAL_SCHEDULER,
- LCM_SCHEDULER,
- BONG_TANGENT_SCHEDULER,
- LTX2_SCHEDULER,
- LOGIT_NORMAL_SCHEDULER,
- FLUX2_SCHEDULER,
- FLUX_SCHEDULER,
- BETA_SCHEDULER,
- LLADA_IMAGE_SCHEDULER,
- SCHEDULER_COUNT
-};
-
-extern SD_API const char* scheduler_to_str[];
-
-enum prediction_t {
- EPS_PRED,
- V_PRED,
- EDM_V_PRED,
- FLOW_PRED,
- FLUX_FLOW_PRED,
- SEFI_FLOW_PRED,
- MINIT2I_FLOW_PRED,
- SENSENOVA_U1_FLOW_PRED,
- FLUX3_FLOW_PRED,
- H3_AV_FLOW_PRED,
- MING_IMAGE_FLOW_PRED,
- PREDICTION_COUNT
-};
-
-// same as enum ggml_type
-enum sd_type_t {
- SD_TYPE_F32 = 0,
- SD_TYPE_F16 = 1,
- SD_TYPE_Q4_0 = 2,
- SD_TYPE_Q4_1 = 3,
- // SD_TYPE_Q4_2 = 4, support has been removed
- // SD_TYPE_Q4_3 = 5, support has been removed
- SD_TYPE_Q5_0 = 6,
- SD_TYPE_Q5_1 = 7,
- SD_TYPE_Q8_0 = 8,
- SD_TYPE_Q8_1 = 9,
- SD_TYPE_Q2_K = 10,
- SD_TYPE_Q3_K = 11,
- SD_TYPE_Q4_K = 12,
- SD_TYPE_Q5_K = 13,
- SD_TYPE_Q6_K = 14,
- SD_TYPE_Q8_K = 15,
- SD_TYPE_IQ2_XXS = 16,
- SD_TYPE_IQ2_XS = 17,
- SD_TYPE_IQ3_XXS = 18,
- SD_TYPE_IQ1_S = 19,
- SD_TYPE_IQ4_NL = 20,
- SD_TYPE_IQ3_S = 21,
- SD_TYPE_IQ2_S = 22,
- SD_TYPE_IQ4_XS = 23,
- SD_TYPE_I8 = 24,
- SD_TYPE_I16 = 25,
- SD_TYPE_I32 = 26,
- SD_TYPE_I64 = 27,
- SD_TYPE_F64 = 28,
- SD_TYPE_IQ1_M = 29,
- SD_TYPE_BF16 = 30,
- // SD_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
- // SD_TYPE_Q4_0_4_8 = 32,
- // SD_TYPE_Q4_0_8_8 = 33,
- SD_TYPE_TQ1_0 = 34,
- SD_TYPE_TQ2_0 = 35,
- // SD_TYPE_IQ4_NL_4_4 = 36,
- // SD_TYPE_IQ4_NL_4_8 = 37,
- // SD_TYPE_IQ4_NL_8_8 = 38,
- SD_TYPE_MXFP4 = 39, // MXFP4 (1 block)
- SD_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
- SD_TYPE_Q1_0 = 41,
- SD_TYPE_Q2_0 = 42,
- SD_TYPE_F8_E4M3 = 43,
- SD_TYPE_F8_E5M2 = 44,
- SD_TYPE_COUNT = 45,
-};
-
-enum sd_log_level_t {
- SD_LOG_DEBUG,
- SD_LOG_VERBOSE,
- SD_LOG_INFO,
- SD_LOG_WARN,
- SD_LOG_ERROR
-};
-
-enum preview_t {
- PREVIEW_NONE,
- PREVIEW_PROJ,
- PREVIEW_TAE,
- PREVIEW_VAE,
- PREVIEW_COUNT
-};
-
-enum lora_apply_mode_t {
- LORA_APPLY_AUTO,
- LORA_APPLY_IMMEDIATELY,
- LORA_APPLY_AT_RUNTIME,
- LORA_APPLY_MODE_COUNT,
-};
-
-typedef struct {
- bool enabled;
- bool temporal_tiling;
- // Spatial tile dimensions in image pixels for both encode and decode; 0 uses 256.
- int tile_size_w;
- int tile_size_h;
- float target_overlap;
- // Positive values override tile_size: <= 1 is a dimension fraction, > 1 a target tile count.
- float rel_size_w;
- float rel_size_h;
- const char* extra_tiling_args;
-} sd_tiling_params_t;
-
-typedef struct {
- const char* name;
- const char* path;
-} sd_embedding_t;
+#ifdef __cplusplus
+extern "C" {
+#endif
-enum sd_vae_format_t {
- SD_VAE_FORMAT_AUTO = -1,
- SD_VAE_FORMAT_FLUX,
- SD_VAE_FORMAT_SD3,
- SD_VAE_FORMAT_FLUX2,
- SD_VAE_FORMAT_WAN,
- SD_VAE_FORMAT_COUNT,
-};
+// Opaque handles
-typedef struct {
- const char* model_path;
- const char* clip_l_path;
- const char* clip_g_path;
- const char* clip_vision_path;
- const char* t5xxl_path;
- const char* llm_path;
- const char* llm_vision_path;
- const char* diffusion_model_path;
- const char* high_noise_diffusion_model_path;
- const char* uncond_diffusion_model_path;
- const char* embeddings_connectors_path;
- const char* vae_path;
- const char* audio_vae_path;
- const char* audio_encoder_path;
- const char* taesd_path;
- const char* control_net_path;
- const char* ip_adapter_path;
- const char* motion_module_path;
- const sd_embedding_t* embeddings;
- uint32_t embedding_count;
- const char* photo_maker_path;
- const char* pulid_weights_path;
- const char* tensor_type_rules;
- int n_threads;
- enum sd_type_t wtype;
- enum rng_type_t rng_type;
- enum rng_type_t sampler_rng_type;
- enum prediction_t prediction;
- enum lora_apply_mode_t lora_apply_mode;
- bool enable_mmap;
- bool flash_attn;
- bool diffusion_flash_attn;
- bool tae_preview_only;
- bool diffusion_conv_direct;
- bool vae_conv_direct;
- bool force_sdxl_vae_conv_scale;
- enum sd_vae_format_t vae_format;
- const char* max_vram; // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget
- bool disable_prefetch; // Disable asynchronous next-segment weight prefetch
- bool eager_load; // Load all params into the params backend at model-load time instead of lazily on first use
- const char* backend;
- const char* params_backend;
- const char* split_mode; // weight distribution for multi-device modules: layer (default) or row, or per-module assignments e.g. "diffusion=row"
- bool auto_fit;
- const char* rpc_servers;
- const char* model_args;
- bool disable_segmented_compute; // Force monolithic graph execution even when automatic graph cutting would fit memory better
- bool batched_cfg; // Run the conditional and unconditional CFG branches in one batched UNet forward when supported
- float linear_scale; // Override linear input scaling; 0 keeps the model default
- float attn_scale; // Override flash-attention K/V scaling; 0 keeps the model default
- const char* tokenizer; // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens
- bool sage_attn;
- int conditioning_cache_size; // Maximum cached conditioning entries per context; 0 disables caching (default: 4)
- const char* flash_attn_mode; // on/off/auto or module assignments (e.g. "auto,te=off"); NULL/empty honors legacy flags.
- const char* conv_direct_mode; // on/off/auto or diffusion/vae/controlnet assignments; NULL/empty honors legacy flags.
-} sd_ctx_params_t;
+typedef struct sd_tensor_t sd_tensor_t;
+typedef struct sd_ctx_t sd_ctx_t;
+typedef struct sd_result sd_result_t;
+typedef struct sd_opt sd_opt_t;
+typedef struct sd_upscaler sd_upscaler_t;
+typedef struct sd_vid_upscaler sd_vid_upscaler_t;
+typedef struct sd_adetailer sd_adetailer_t;
+
+// Status and diagnostics
+
+typedef enum {
+ SD_STATUS_OK = 0,
+ SD_STATUS_INVALID_ARGUMENT = 1,
+ SD_STATUS_UNKNOWN_KEY = 2,
+ SD_STATUS_TYPE_MISMATCH = 3,
+ SD_STATUS_OUT_OF_RANGE = 4,
+ SD_STATUS_UNSUPPORTED = 5,
+ SD_STATUS_OUT_OF_MEMORY = 6,
+ SD_STATUS_LOAD_FAILED = 7,
+ SD_STATUS_GENERATION_FAILED = 8,
+ SD_STATUS_BUSY = 9,
+ SD_STATUS_CANCELLED = 10,
+} sd_status_t;
+
+// Thread-local diagnostic, cleared by successful status/accessor calls.
+// Valid until the next API call; free functions and tensor retain preserve it.
+SD_API const char* sd_get_last_error(void);
+
+// Tensors and media
+
+typedef int32_t sd_tensor_type_t;
+#define SD_TENSOR_U8 1
+#define SD_TENSOR_I32 2
+#define SD_TENSOR_I64 3
+#define SD_TENSOR_F32 4
+#define SD_TENSOR_F64 5
typedef struct {
uint32_t sample_rate;
- uint32_t channels;
- uint64_t sample_count;
- float* data;
+ const sd_tensor_t* data;
} sd_audio_t;
typedef struct {
- uint32_t width;
- uint32_t height;
- uint32_t channel;
- uint8_t* data;
-} sd_image_t;
-
-typedef struct {
- sd_image_t image;
- // Owned UTF-8 JSON using the sdcpp.generation/v1 schema.
- char* generation_info;
-} sd_image_result_t;
-
-typedef struct {
- sd_image_result_t* items;
- int count;
-} sd_image_generation_result_t;
-
-typedef struct {
- sd_image_t* frames;
- int frame_count;
- sd_audio_t* audio;
+ const sd_tensor_t* data;
int fps;
- char* generation_info;
-} sd_video_generation_result_t;
+ sd_audio_t audio;
+} sd_video_t;
+
+// Contiguous dimension first. Images: U8 [C,W,H], audio: F32 [C,N], video: U8 [C,W,H,T].
+// create copies data; data_size must exactly match type and shape. Rank <= 8.
+// Rank zero is a scalar; zero dimensions represent empty tensors.
+SD_API sd_tensor_t* sd_tensor_create(
+ sd_tensor_type_t type,
+ size_t rank,
+ const int64_t* shape,
+ const void* data,
+ size_t data_size);
+
+// retain returns the same immutable handle with one additional ownership reference.
+SD_API sd_tensor_t* sd_tensor_retain(const sd_tensor_t* tensor);
+SD_API void sd_tensor_free(sd_tensor_t* tensor);
+SD_API sd_tensor_type_t sd_tensor_type(const sd_tensor_t* tensor);
+SD_API size_t sd_tensor_rank(const sd_tensor_t* tensor);
+SD_API int64_t sd_tensor_dim(const sd_tensor_t* tensor, size_t axis);
+SD_API size_t sd_tensor_byte_size(const sd_tensor_t* tensor);
+SD_API const void* sd_tensor_data(const sd_tensor_t* tensor);
+
+// Options
typedef struct {
- // Semicolon-separated target=...,key=value rules. NULL preserves defaults.
- const char* rules;
-} sd_image_preprocess_params_t;
+ const char* name;
+ const char* path;
+} sd_embedding_t;
typedef struct {
- sd_image_t* frames;
- int frame_count;
- int fps;
- sd_audio_t audio;
-} sd_ref_video_t;
+ bool is_high_noise;
+ float multiplier;
+ const char* path;
+} sd_lora_t;
-typedef struct {
- int* layers;
- size_t layer_count;
- float layer_start;
- float layer_end;
- float scale;
-} sd_slg_params_t;
+typedef sd_opt_t sd_ctx_opt_t;
+typedef sd_opt_t sd_img_gen_opt_t;
+typedef sd_opt_t sd_vid_gen_opt_t;
+typedef sd_opt_t sd_upscaler_opt_t;
+typedef sd_opt_t sd_upscale_opt_t;
+typedef sd_opt_t sd_vid_upscaler_opt_t;
+typedef sd_opt_t sd_vid_upscale_opt_t;
+typedef sd_opt_t sd_adetailer_opt_t;
+typedef sd_opt_t sd_adetail_opt_t;
+
+// Parameter keys and types are documented in docs/guides/c_api.md.
+// Setters replace values atomically; a failed setter preserves the previous value.
+// Strings, arrays and media descriptors are copied. Tensor storage is retained;
+// callers may free their tensor handles after a successful setter.
+// Do not mutate/free params concurrently with an operation using them.
+// The aliases share one C type; constructors bind their purpose at runtime.
+SD_API sd_ctx_opt_t* sd_ctx_opt_create(void);
+SD_API sd_img_gen_opt_t* sd_img_gen_opt_create(void);
+SD_API sd_vid_gen_opt_t* sd_vid_gen_opt_create(void);
+SD_API sd_upscaler_opt_t* sd_upscaler_opt_create(void);
+SD_API sd_upscale_opt_t* sd_upscale_opt_create(void);
+SD_API sd_vid_upscaler_opt_t* sd_vid_upscaler_opt_create(void);
+SD_API sd_vid_upscale_opt_t* sd_vid_upscale_opt_create(void);
+SD_API sd_adetailer_opt_t* sd_adetailer_opt_create(void);
+SD_API sd_adetail_opt_t* sd_adetail_opt_create(void);
+SD_API void sd_opt_free(sd_opt_t* params);
+
+SD_API sd_status_t sd_opt_set_bool(sd_opt_t* params, const char* key, bool value);
+SD_API sd_status_t sd_opt_set_int(sd_opt_t* params, const char* key, int64_t value);
+SD_API sd_status_t sd_opt_set_float(sd_opt_t* params, const char* key, double value);
+SD_API sd_status_t sd_opt_set_string(sd_opt_t* params, const char* key, const char* value);
+SD_API sd_status_t sd_opt_set_tensor(sd_opt_t* params, const char* key, const sd_tensor_t* tensor);
+SD_API sd_status_t sd_opt_set_tensors(
+ sd_opt_t* params,
+ const char* key,
+ const sd_tensor_t* const* tensors,
+ size_t count);
+SD_API sd_status_t sd_opt_set_floats(sd_opt_t* params, const char* key, const float* values, size_t count);
+SD_API sd_status_t sd_opt_set_ints(sd_opt_t* params, const char* key, const int32_t* values, size_t count);
+SD_API sd_status_t sd_opt_set_loras(sd_opt_t* params, const char* key, const sd_lora_t* values, size_t count);
+SD_API sd_status_t sd_opt_set_embeddings(
+ sd_opt_t* params,
+ const char* key,
+ const sd_embedding_t* values,
+ size_t count);
+SD_API sd_status_t sd_opt_set_audios(sd_opt_t* params, const char* key, const sd_audio_t* values, size_t count);
+SD_API sd_status_t sd_opt_set_videos(sd_opt_t* params, const char* key, const sd_video_t* values, size_t count);
+
+// Setters/unset reject keys for another purpose. Unsetting an applicable known
+// absent key succeeds; unknown keys fail. NULL is not a string value.
+SD_API sd_status_t sd_opt_unset(sd_opt_t* params, const char* key);
+
+// Context and generation
+
+// Output pointers are required and set to NULL on failure. Context creation retains
+// its own configuration; params may be freed immediately after the call returns.
+SD_API sd_status_t sd_ctx_create(const sd_ctx_opt_t* params, sd_ctx_t** ctx_out);
+SD_API void sd_ctx_free(sd_ctx_t* ctx);
+
+SD_API sd_status_t sd_generate_image(sd_ctx_t* ctx, const sd_img_gen_opt_t* params, sd_result_t** result_out);
+SD_API sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* params, sd_result_t** result_out);
-typedef struct {
- float txt_cfg;
- float img_cfg;
- float distilled_guidance;
- sd_slg_params_t slg;
-} sd_guidance_params_t;
+SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
+SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
-typedef struct {
- sd_guidance_params_t guidance;
- enum scheduler_t scheduler;
- enum sample_method_t sample_method;
- int sample_steps;
- float eta;
- int shifted_timestep;
- float* custom_sigmas;
- int custom_sigmas_count;
- float flow_shift;
- const char* extra_sample_args;
-} sd_sample_params_t;
+// ControlNet hot-swap APIs are not safe to call while generation is in flight.
+SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path);
+SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx);
+SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx);
-typedef struct {
- sd_image_t* id_images;
- int id_images_count;
- const char* id_embed_path;
- float style_strength;
-} sd_pm_params_t; // photo maker
+// Requires a loaded context; returns a static string owned by the library, or "Unknown".
+SD_API const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx);
-typedef struct {
- const char* id_embedding_path;
- float id_weight;
-} sd_pulid_params_t;
-
-enum sd_cache_mode_t {
- SD_CACHE_DISABLED = 0,
- SD_CACHE_EASYCACHE,
- SD_CACHE_UCACHE,
- SD_CACHE_DBCACHE,
- SD_CACHE_TAYLORSEER,
- SD_CACHE_CACHE_DIT,
- SD_CACHE_SPECTRUM,
+enum sd_cancel_mode_t {
+ // Stop the current generation as soon as possible.
+ SD_CANCEL_ALL,
+ // Finish the current image sample, then skip additional batch latents and return completed images.
+ SD_CANCEL_NEW_LATENTS,
+ // Clear a pending cancellation request.
+ SD_CANCEL_RESET
};
-typedef struct {
- enum sd_cache_mode_t mode;
- float reuse_threshold;
- float start_percent;
- float end_percent;
- float error_decay_rate;
- bool use_relative_threshold;
- bool reset_error_on_compute;
- int Fn_compute_blocks;
- int Bn_compute_blocks;
- float residual_diff_threshold;
- int max_warmup_steps;
- int max_cached_steps;
- int max_continuous_cached_steps;
- int taylorseer_n_derivatives;
- int taylorseer_skip_interval;
- const char* scm_mask;
- bool scm_policy_dynamic;
- float spectrum_w;
- int spectrum_m;
- float spectrum_lam;
- int spectrum_window_size;
- float spectrum_flex_window;
- int spectrum_warmup_steps;
- float spectrum_stop_percent;
-} sd_cache_params_t;
+SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode);
-typedef struct {
- bool is_high_noise;
- float multiplier;
- const char* path;
-} sd_lora_t;
+// Results
-enum sd_hires_upscaler_t {
- SD_HIRES_UPSCALER_NONE,
- SD_HIRES_UPSCALER_LATENT,
- SD_HIRES_UPSCALER_LATENT_NEAREST,
- SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT,
- SD_HIRES_UPSCALER_LATENT_ANTIALIASED,
- SD_HIRES_UPSCALER_LATENT_BICUBIC,
- SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED,
- SD_HIRES_UPSCALER_LANCZOS,
- SD_HIRES_UPSCALER_NEAREST,
- SD_HIRES_UPSCALER_MODEL,
- SD_HIRES_UPSCALER_COUNT,
-};
+typedef enum {
+ SD_RESULT_INVALID = 0,
+ SD_RESULT_IMAGE = 1,
+ SD_RESULT_VIDEO = 2,
+} sd_result_kind_t;
-enum sd_control_type_t {
- SD_CONTROL_TYPE_NONE,
- SD_CONTROL_TYPE_OPENPOSE,
- SD_CONTROL_TYPE_DEPTH,
- SD_CONTROL_TYPE_SCRIBBLE, // thick lines: scribble, hed, softedge, ted
- SD_CONTROL_TYPE_CANNY, // thin lines: canny, lineart, anime lineart, mlsd
- SD_CONTROL_TYPE_NORMAL,
- SD_CONTROL_TYPE_SEGMENT,
- SD_CONTROL_TYPE_TILE,
- SD_CONTROL_TYPE_REPAINT, // inpaint, outpaint
- SD_CONTROL_TYPE_COUNT,
-};
+// Results own all outputs independently of ctx and params. Views are read-only and
+// borrowed until sd_result_free(). Retain a tensor to keep its data independently.
+// Media descriptors contain metadata and borrowed immutable tensor handles.
+SD_API sd_result_kind_t sd_result_get_kind(const sd_result_t* result);
+SD_API size_t sd_result_get_image_count(const sd_result_t* result);
+SD_API const sd_tensor_t* sd_result_get_image(const sd_result_t* result, size_t index);
-typedef struct {
- bool enabled;
- enum sd_hires_upscaler_t upscaler;
- const char* model_path;
- float scale;
- int target_width;
- int target_height;
- int steps;
- float denoising_strength;
- int upscale_tile_size;
- float* custom_sigmas;
- int custom_sigmas_count;
-} sd_hires_params_t;
+// Materializes contiguous U8 [C,W,H,T] storage on first access.
+SD_API const sd_video_t* sd_result_get_video(const sd_result_t* result);
-typedef struct {
- const sd_lora_t* loras;
- uint32_t lora_count;
- const char* prompt;
- const char* negative_prompt;
- int clip_skip;
- sd_image_t init_image;
- sd_image_t* ref_images;
- int ref_images_count;
- const char* ref_image_args;
- sd_image_t mask_image;
- int width;
- int height;
- sd_sample_params_t sample_params;
- float strength;
- int64_t seed;
- int batch_count;
- sd_image_t control_image;
- float control_strength;
- sd_image_t ip_adapter_image;
- float ip_adapter_strength;
- sd_pm_params_t pm_params;
- sd_pulid_params_t pulid_params;
- sd_tiling_params_t vae_tiling_params;
- sd_cache_params_t cache;
- sd_hires_params_t hires;
- int qwen_image_layers;
- bool circular_x;
- bool circular_y;
- sd_image_preprocess_params_t image_preprocess;
- enum sd_control_type_t control_type;
-} sd_img_gen_params_t;
+// Image metadata is per image; video metadata is shared by all frame indices.
+SD_API const char* sd_result_get_generation_info(const sd_result_t* result, size_t index);
+SD_API const sd_audio_t* sd_result_get_audio(const sd_result_t* result);
+SD_API int sd_result_get_fps(const sd_result_t* result);
-typedef struct {
- const sd_lora_t* loras;
- uint32_t lora_count;
- const char* prompt;
- const char* negative_prompt;
- int clip_skip;
- sd_image_t init_image;
- sd_image_t end_image;
- sd_image_t* ref_images;
- int ref_images_count;
- sd_ref_video_t* ref_videos;
- int ref_videos_count;
- sd_audio_t* ref_audios;
- int ref_audios_count;
- sd_image_t* control_frames;
- int control_frames_size;
- int width;
- int height;
- sd_sample_params_t sample_params;
- sd_sample_params_t high_noise_sample_params;
- float moe_boundary;
- float strength;
- int64_t seed;
- int video_frames;
- int fps;
- float vace_strength;
- sd_tiling_params_t vae_tiling_params;
- sd_cache_params_t cache;
- sd_hires_params_t hires;
- bool circular_x;
- bool circular_y;
- sd_image_preprocess_params_t image_preprocess;
- const char* ref_image_args;
-} sd_vid_gen_params_t;
+SD_API void sd_result_free(sd_result_t* result);
-typedef struct sd_ctx_t sd_ctx_t;
-struct ggml_tensor;
+// Callbacks and progress
+
+enum sd_log_level_t {
+ SD_LOG_DEBUG,
+ SD_LOG_VERBOSE,
+ SD_LOG_INFO,
+ SD_LOG_WARN,
+ SD_LOG_ERROR
+};
typedef void (*sd_log_cb_t)(enum sd_log_level_t level, const char* text, void* data);
+SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
+
typedef void (*sd_progress_cb_t)(int step, int steps, float time, void* data);
+SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
+
typedef struct {
const char* stage;
const char* detail;
@@ -509,222 +244,174 @@ typedef struct {
int64_t total;
} sd_generation_progress_t;
typedef void (*sd_generation_progress_cb_t)(const sd_generation_progress_t* progress, void* data);
-typedef void (*sd_preview_cb_t)(int step, int frame_count, sd_image_t* frames, bool is_noisy, void* data);
+
+// Context-local generation events, independent of preview and the legacy progress callback.
+// Strings and the event are valid only during the synchronous callback. Callbacks must not throw.
+// total == 0 means unknown; sample_pass counts from 1 since callback registration.
+// Register/clear only while the context is idle. Passing NULL clears the callback.
+SD_API void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data);
typedef struct {
int sample_pass;
int total_steps;
} sd_preview_info_t;
-typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data);
-// Resolves <image:source> markers. Pixels are borrowed and copied before the
-// next callback; the caller retains ownership. Register/clear while ctx is idle.
-typedef bool (*sd_prompt_image_cb_t)(const char* source, void* data, sd_image_t* image_out);
-SD_API void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data);
+// Frames are borrowed during the callback; retain a tensor to keep its snapshot.
+typedef void (*sd_preview_cb_t)(int step, int frame_count, const sd_tensor_t* const* frames, bool is_noisy, void* data);
-SD_API void sd_set_log_callback(sd_log_cb_t sd_log_cb, void* data);
-SD_API void sd_set_progress_callback(sd_progress_cb_t cb, void* data);
-// Context-local generation events, independent of preview and the legacy progress callback.
-// Strings and the event are valid only during the synchronous callback. Callbacks must not throw.
-// total == 0 means unknown; sample_pass counts from 1 since callback registration.
-// Register/clear only while the context is idle. Passing nullptr clears the callback.
-SD_API void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data);
// In each sampling pass, a positive interval previews every Nth denoiser step, while a
// negative interval previews only completed logical step -interval. Zero previews the final
// completed step of the first sampling pass (base-resolution or high-noise).
-SD_API void sd_set_preview_callback(sd_preview_cb_t cb, enum preview_t mode, int interval, bool denoised, bool noisy, void* data);
+// Mode is "none", "proj", "tae" or "vae"; invalid input preserves registration.
+SD_API sd_status_t sd_set_preview_callback(
+ sd_preview_cb_t cb,
+ const char* mode,
+ int interval,
+ bool denoised,
+ bool noisy,
+ void* data);
+
// Query from a preview callback. Passes are numbered from 1 since the last
// sd_set_preview_callback call; total_steps is the actual count in that pass.
// Both fields are zero before sampling starts.
SD_API sd_preview_info_t sd_get_preview_info();
-SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data);
-SD_API int32_t sd_get_num_physical_cores();
-SD_API const char* sd_get_system_info();
-SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx);
-SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx);
-
-// ControlNet hot-swap APIs are not safe to call while generation is in flight.
-SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path);
-SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx);
-SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx);
-SD_API const char* sd_type_name(enum sd_type_t type);
-SD_API enum sd_type_t str_to_sd_type(const char* str);
-SD_API const char* sd_rng_type_name(enum rng_type_t rng_type);
-SD_API enum rng_type_t str_to_rng_type(const char* str);
-SD_API const char* sd_sample_method_name(enum sample_method_t sample_method);
-SD_API enum sample_method_t str_to_sample_method(const char* str);
-SD_API const char* sd_scheduler_name(enum scheduler_t scheduler);
-SD_API enum scheduler_t str_to_scheduler(const char* str);
-SD_API const char* sd_prediction_name(enum prediction_t prediction);
-SD_API enum prediction_t str_to_prediction(const char* str);
-SD_API const char* sd_preview_name(enum preview_t preview);
-SD_API enum preview_t str_to_preview(const char* str);
-SD_API const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode);
-SD_API enum lora_apply_mode_t str_to_lora_apply_mode(const char* str);
-SD_API const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler);
-SD_API enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str);
-SD_API const char* sd_control_type_name(enum sd_control_type_t control_type);
-// Also accepts the names of the kinds a type stands for (hed, lineart, mlsd, ...);
-// returns SD_CONTROL_TYPE_COUNT when the name is unknown.
-SD_API enum sd_control_type_t str_to_sd_control_type(const char* str);
-
-SD_API void sd_cache_params_init(sd_cache_params_t* cache_params);
-SD_API void sd_hires_params_init(sd_hires_params_t* hires_params);
-
-SD_API void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params);
-SD_API char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params);
-
-SD_API sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params);
-SD_API void free_sd_ctx(sd_ctx_t* sd_ctx);
-SD_API void free_sd_audio(sd_audio_t* audio);
-
-SD_API void sd_sample_params_init(sd_sample_params_t* sample_params);
-SD_API char* sd_sample_params_to_str(const sd_sample_params_t* sample_params);
+// Resolves <image:source> markers. The returned U8 [C,W,H] tensor is borrowed
+// and copied before the next callback. Register/clear while ctx is idle.
+typedef bool (*sd_prompt_image_cb_t)(const char* source, void* data, const sd_tensor_t** image_out);
+SD_API void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data);
-// Requires a loaded context; returns a static string owned by the library, or "Unknown".
-SD_API const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx);
+struct ggml_tensor;
-SD_API enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx);
-SD_API enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_method_t sample_method);
+typedef bool (*sd_graph_eval_callback_t)(struct ggml_tensor* t, bool ask, void* user_data);
+SD_API void sd_set_backend_eval_callback(sd_graph_eval_callback_t cb, void* data);
-SD_API void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params);
-SD_API char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params);
-SD_API bool generate_image(sd_ctx_t* sd_ctx,
- const sd_img_gen_params_t* sd_img_gen_params,
- sd_image_generation_result_t** result_out);
+// Image upscaling
-// Results own their pixels, audio and generation_info, independently of the context.
-// Generation calls require result_out and set it to NULL on failure.
-SD_API void free_sd_image_generation_result(sd_image_generation_result_t* result);
-SD_API void free_sd_video_generation_result(sd_video_generation_result_t* result);
+SD_API sd_status_t sd_upscaler_create(const sd_upscaler_opt_t* opt, sd_upscaler_t** out);
+SD_API void sd_upscaler_free(sd_upscaler_t* upscaler);
+SD_API sd_status_t sd_upscale(sd_upscaler_t* upscaler, const sd_upscale_opt_t* opt, sd_result_t** out);
+SD_API int sd_upscaler_get_scale(const sd_upscaler_t* upscaler);
-enum sd_cancel_mode_t {
- // Stop the current generation as soon as possible.
- SD_CANCEL_ALL,
- // Finish the current image sample, then skip additional batch latents and return completed images.
- SD_CANCEL_NEW_LATENTS,
- // Clear a pending cancellation request.
- SD_CANCEL_RESET
-};
+// Reads model metadata only; returns 0 if it is not a recognized RGB upscaler.
+SD_API int sd_upscaler_probe_scale(const char* model_path);
-SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode);
+// Video upscaling
-SD_API void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params);
-SD_API bool generate_video(sd_ctx_t* sd_ctx,
- const sd_vid_gen_params_t* sd_vid_gen_params,
- sd_video_generation_result_t** result_out);
+SD_API sd_status_t sd_vid_upscaler_create(const sd_vid_upscaler_opt_t* opt, sd_vid_upscaler_t** out);
+SD_API void sd_vid_upscaler_free(sd_vid_upscaler_t* upscaler);
+SD_API sd_status_t sd_upscale_video(
+ sd_vid_upscaler_t* upscaler,
+ const sd_vid_upscale_opt_t* opt,
+ sd_result_t** out);
-typedef struct upscaler_ctx_t upscaler_ctx_t;
+// ADetailer
-SD_API upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path,
- bool direct,
- int n_threads,
- int tile_size,
- const char* backend,
- const char* params_backend);
-SD_API void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx);
+SD_API sd_status_t sd_adetailer_create(const sd_adetailer_opt_t* opt, sd_adetailer_t** out);
+SD_API void sd_adetailer_free(sd_adetailer_t* adetailer);
+SD_API sd_status_t sd_adetail(
+ sd_adetailer_t* adetailer,
+ sd_ctx_t* ctx,
+ const sd_adetail_opt_t* opt,
+ sd_result_t** out);
-SD_API bool upscale(upscaler_ctx_t* upscaler_ctx,
- sd_image_t input_image,
- uint32_t upscale_factor,
- sd_image_generation_result_t** result_out);
+// Image preprocessing
-SD_API int get_upscale_factor(upscaler_ctx_t* upscaler_ctx);
-// Reads model metadata only; returns 0 if the file is not a recognized RGB ESRGAN model.
-SD_API int get_upscaler_model_scale(const char* model_path);
+SD_API sd_status_t sd_preprocess_canny(
+ const sd_tensor_t* image,
+ float high_threshold,
+ float low_threshold,
+ float weak,
+ float strong,
+ bool inverse,
+ sd_tensor_t** image_out);
-typedef struct video_upscaler_ctx_t video_upscaler_ctx_t;
+// Weight types and model conversion
-typedef struct {
- const char* diffusion_model_path;
- const char* vae_path;
- // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints.
- int upscale_factor;
- int n_threads;
- const char* backend;
- const char* params_backend;
- // auto/on/off or diffusion/vae assignments; null selects auto.
- const char* flash_attn_mode;
- const char* conv_direct_mode;
-} sd_video_upscaler_ctx_params_t;
+// same as enum ggml_type
+enum sd_type_t {
+ SD_TYPE_F32 = 0,
+ SD_TYPE_F16 = 1,
+ SD_TYPE_Q4_0 = 2,
+ SD_TYPE_Q4_1 = 3,
+ // SD_TYPE_Q4_2 = 4, support has been removed
+ // SD_TYPE_Q4_3 = 5, support has been removed
+ SD_TYPE_Q5_0 = 6,
+ SD_TYPE_Q5_1 = 7,
+ SD_TYPE_Q8_0 = 8,
+ SD_TYPE_Q8_1 = 9,
+ SD_TYPE_Q2_K = 10,
+ SD_TYPE_Q3_K = 11,
+ SD_TYPE_Q4_K = 12,
+ SD_TYPE_Q5_K = 13,
+ SD_TYPE_Q6_K = 14,
+ SD_TYPE_Q8_K = 15,
+ SD_TYPE_IQ2_XXS = 16,
+ SD_TYPE_IQ2_XS = 17,
+ SD_TYPE_IQ3_XXS = 18,
+ SD_TYPE_IQ1_S = 19,
+ SD_TYPE_IQ4_NL = 20,
+ SD_TYPE_IQ3_S = 21,
+ SD_TYPE_IQ2_S = 22,
+ SD_TYPE_IQ4_XS = 23,
+ SD_TYPE_I8 = 24,
+ SD_TYPE_I16 = 25,
+ SD_TYPE_I32 = 26,
+ SD_TYPE_I64 = 27,
+ SD_TYPE_F64 = 28,
+ SD_TYPE_IQ1_M = 29,
+ SD_TYPE_BF16 = 30,
+ // SD_TYPE_Q4_0_4_4 = 31, support has been removed from gguf files
+ // SD_TYPE_Q4_0_4_8 = 32,
+ // SD_TYPE_Q4_0_8_8 = 33,
+ SD_TYPE_TQ1_0 = 34,
+ SD_TYPE_TQ2_0 = 35,
+ // SD_TYPE_IQ4_NL_4_4 = 36,
+ // SD_TYPE_IQ4_NL_4_8 = 37,
+ // SD_TYPE_IQ4_NL_8_8 = 38,
+ SD_TYPE_MXFP4 = 39, // MXFP4 (1 block)
+ SD_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
+ SD_TYPE_Q1_0 = 41,
+ SD_TYPE_Q2_0 = 42,
+ SD_TYPE_F8_E4M3 = 43,
+ SD_TYPE_F8_E5M2 = 44,
+ SD_TYPE_COUNT = 45,
+};
-typedef struct {
- const sd_image_t* frames;
- int frame_count;
- int fps;
- int64_t seed;
- // RelayVSR reference window, including both endpoints; defaults to 16.
- int window_size;
- // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5.
- int temporal_chunk_size;
-} sd_video_upscale_params_t;
-
-SD_API void sd_video_upscaler_ctx_params_init(sd_video_upscaler_ctx_params_t* params);
-SD_API void sd_video_upscale_params_init(sd_video_upscale_params_t* params);
-SD_API video_upscaler_ctx_t* new_video_upscaler_ctx(const sd_video_upscaler_ctx_params_t* params);
-SD_API void free_video_upscaler_ctx(video_upscaler_ctx_t* ctx);
-// Frames are RGB, have equal dimensions, and remain owned by the caller.
-// Free the result with free_sd_video_generation_result().
-SD_API bool upscale_video(video_upscaler_ctx_t* ctx,
- const sd_video_upscale_params_t* params,
- sd_video_generation_result_t** result_out);
-
-typedef struct adetailer_ctx_t adetailer_ctx_t;
+SD_API const char* sd_type_name(enum sd_type_t type);
+SD_API enum sd_type_t str_to_sd_type(const char* str);
-typedef struct {
- const char* prompt;
- const char* negative_prompt;
- const char* extra_ad_args;
-} sd_adetailer_params_t;
-
-SD_API adetailer_ctx_t* new_adetailer_ctx(const char* detector_path,
- int n_threads,
- const char* backend,
- const char* params_backend);
-SD_API void free_adetailer_ctx(adetailer_ctx_t* adetailer_ctx);
-SD_API bool adetail_image(adetailer_ctx_t* adetailer_ctx,
- sd_ctx_t* sd_ctx,
- sd_image_t input_image,
- const sd_adetailer_params_t* adetailer_params,
- const sd_img_gen_params_t* inpaint_params,
- sd_image_generation_result_t** result_out);
-
-SD_API bool convert(const char* input_path,
- const char* vae_path,
- const char* output_path,
- enum sd_type_t output_type,
- const char* tensor_type_rules,
- bool convert_name);
-
-SD_API bool convert_with_components(const char* model_path,
- const char* clip_l_path,
- const char* clip_g_path,
- const char* t5xxl_path,
- const char* diffusion_model_path,
- const char* vae_path,
- const char* output_path,
- enum sd_type_t output_type,
- const char* tensor_type_rules,
- bool convert_name,
- int n_threads,
- const sd_lora_t* loras,
- int lora_count);
-
-SD_API bool preprocess_canny(sd_image_t image,
- float high_threshold,
- float low_threshold,
- float weak,
- float strong,
- bool inverse);
+SD_API bool convert(
+ const char* input_path,
+ const char* vae_path,
+ const char* output_path,
+ enum sd_type_t output_type,
+ const char* tensor_type_rules,
+ bool convert_name);
+
+SD_API bool convert_with_components(
+ const char* model_path,
+ const char* clip_l_path,
+ const char* clip_g_path,
+ const char* t5xxl_path,
+ const char* diffusion_model_path,
+ const char* vae_path,
+ const char* output_path,
+ enum sd_type_t output_type,
+ const char* tensor_type_rules,
+ bool convert_name,
+ int n_threads,
+ const sd_lora_t* loras,
+ int lora_count);
+
+// Importance matrix
SD_API bool load_imatrix(const char* imatrix_path);
SD_API void save_imatrix(const char* imatrix_path);
SD_API void enable_imatrix_collection(void);
SD_API void disable_imatrix_collection(void);
-SD_API const char* sd_commit(void);
-SD_API const char* sd_version(void);
+// Devices and system information
// List available ggml backend devices, one `name<TAB>description` per line.
// The names are the device names accepted by the --backend / --params-backend
@@ -737,9 +424,13 @@ SD_API size_t sd_list_devices(char* buffer, size_t buffer_size);
// Returns false if RPC is unavailable or an endpoint has no usable devices.
SD_API bool sd_add_rpc_servers(const char* servers);
-// for C API, caller needs to call free_sd_images to free the memory after use
-// This helps avoid CRT problems on Windows when memory is allocated in the library but freed in the caller, which may use a different CRT.
-SD_API void free_sd_images(sd_image_t* result_images, int num_images);
+SD_API int32_t sd_get_num_physical_cores();
+SD_API const char* sd_get_system_info();
+
+// Build information
+
+SD_API const char* sd_commit(void);
+SD_API const char* sd_version(void);
#ifdef __cplusplus
}
diff --git a/scripts/format-code.ps1 b/scripts/format-code.ps1
index 02097e6..782e092 100644
--- a/scripts/format-code.ps1
+++ b/scripts/format-code.ps1
@@ -2,6 +2,9 @@ $patterns = @(
"src/*.cpp"
"src/*.h"
"src/*.hpp"
+ "src/api/*.cpp"
+ "src/api/*.h"
+ "src/api/*.hpp"
"src/conditioning/*.cpp"
"src/conditioning/*.h"
"src/conditioning/*.hpp"
diff --git a/scripts/format-code.sh b/scripts/format-code.sh
index dd4d0fa..c2806f6 100644
--- a/scripts/format-code.sh
+++ b/scripts/format-code.sh
@@ -6,6 +6,7 @@ repo_root="$(cd "$(dirname "$0")/.." && pwd)"
cd "$repo_root"
for f in src/*.cpp src/*.h src/*.hpp \
+ src/api/*.cpp src/api/*.h src/api/*.hpp \
src/conditioning/*.cpp src/conditioning/*.h src/conditioning/*.hpp \
src/core/*.cpp src/core/*.h src/core/*.hpp \
src/core/cpu/*.cpp src/core/cpu/*.h \
diff --git a/src/api/api.cpp b/src/api/api.cpp
new file mode 100644
index 0000000..72cd6d4
--- /dev/null
+++ b/src/api/api.cpp
@@ -0,0 +1,245 @@
+#include "api.h"
+#include "options.h"
+#include "result.h"
+#include "tensor.h"
+
+#include <algorithm>
+#include <cstdio>
+#include <cstring>
+
+#include "core/ggml_extend.h"
+#include "core/option_types.h"
+#include "core/util.h"
+#include "pipeline/diffusion_engine.h"
+#include "pipeline/generation.h"
+
+static thread_local char last_error[512] = {};
+
+void sd::api::clear_error() {
+ last_error[0] = '\0';
+}
+
+sd_status_t sd::api::error(sd_status_t status, const char* key, const char* reason) {
+ snprintf(last_error, sizeof(last_error), "%s%s%s", key ? key : "", key ? ": " : "", reason);
+ return status;
+}
+
+const char* sd_get_last_error(void) {
+ return last_error;
+}
+
+const char* sd_type_name(enum sd_type_t type) {
+ if (type == SD_TYPE_F8_E4M3) {
+ return "f8_e4m3";
+ }
+ if (type == SD_TYPE_F8_E5M2) {
+ return "f8_e5m2";
+ }
+ const auto ggml_type = sd_type_to_ggml_type(type);
+ if (ggml_type != GGML_TYPE_COUNT) {
+ return ggml_type_name(ggml_type);
+ }
+ return "NONE";
+}
+
+enum sd_type_t str_to_sd_type(const char* str) {
+ if (!strcmp(str, "f8_e4m3")) {
+ return SD_TYPE_F8_E4M3;
+ }
+ if (!strcmp(str, "f8_e5m2")) {
+ return SD_TYPE_F8_E5M2;
+ }
+ for (int i = 0; i < std::min<int>(SD_TYPE_COUNT, GGML_TYPE_COUNT); i++) {
+ auto trait = ggml_get_type_traits((ggml_type)i);
+ if (!strcmp(str, trait->type_name)) {
+ return (enum sd_type_t)i;
+ }
+ }
+ return SD_TYPE_COUNT;
+}
+
+sd_status_t sd_set_preview_callback(sd_preview_cb_t cb, const char* mode, int interval, bool denoised, bool noisy, void* data) {
+ sd::api::clear_error();
+ if (!mode)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "preview.mode", "mode is required");
+ const auto parsed = str_to_preview(mode);
+ if (parsed == PREVIEW_COUNT)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "preview.mode", "unknown preview mode");
+ sd_set_preview_callback_impl(cb, parsed, interval, denoised, noisy, data);
+ return SD_STATUS_OK;
+}
+
+void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data) {
+ if (ctx == nullptr || ctx->sd == nullptr) {
+ return;
+ }
+ std::lock_guard<std::recursive_mutex> lock(ctx->sd->execution_mutex);
+ if (!ctx->sd->executing_) {
+ ctx->sd->prompt_image_callback = cb;
+ ctx->sd->prompt_image_callback_data = data;
+ }
+}
+
+void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data) {
+ if (ctx == nullptr || ctx->sd == nullptr) {
+ return;
+ }
+ std::lock_guard<std::recursive_mutex> lock(ctx->sd->execution_mutex);
+ if (!ctx->sd->executing_) {
+ ctx->sd->generation_progress.set_callback(cb, data);
+ }
+}
+
+static sd_ctx_t* create_context(const sd::ContextConfig& params) {
+ auto engine = std::make_unique<StableDiffusionGGML>();
+ if (!engine->init(¶ms))
+ return nullptr;
+ auto context = std::make_unique<sd_ctx_t>();
+ context->sd = engine.release();
+ return context.release();
+}
+
+sd_status_t sd_ctx_create(const sd_ctx_opt_t* params, sd_ctx_t** ctx_out) {
+ if (ctx_out)
+ *ctx_out = nullptr;
+ return sd::api::call([&] {
+ if (!ctx_out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "ctx_out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(params, sd::api::Context, config);
+ const auto& context = config.context;
+ if (status != SD_STATUS_OK)
+ return status;
+ *ctx_out = create_context(context);
+ return *ctx_out ? SD_STATUS_OK : sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "context creation failed; see log for details");
+ },
+ SD_STATUS_LOAD_FAILED);
+}
+
+void sd_ctx_free(sd_ctx_t* sd_ctx) {
+ if (!sd_ctx)
+ return;
+ if (sd_ctx->sd != nullptr) {
+ delete sd_ctx->sd;
+ sd_ctx->sd = nullptr;
+ }
+ delete sd_ctx;
+}
+
+SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) {
+ if (sd_ctx && sd_ctx->sd) {
+ if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET) {
+ mode = SD_CANCEL_ALL;
+ }
+ sd_ctx->sd->set_cancel_flag(mode);
+ }
+}
+
+SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx) {
+ if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+ return false;
+ }
+ return sd_version_supports_image_generation(sd_ctx->sd->version);
+}
+
+SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) {
+ if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+ return false;
+ }
+ if (sd_ctx->sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd_ctx->sd->version)) {
+ return true;
+ }
+ return sd_version_supports_video_generation(sd_ctx->sd->version);
+}
+
+SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) {
+ if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) {
+ return false;
+ }
+ return sd_ctx->sd->load_control_net_from_file(path);
+}
+
+SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx) {
+ if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+ return false;
+ }
+ return sd_ctx->sd->unload_control_net();
+}
+
+SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx) {
+ if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
+ return false;
+ }
+ if (sd_ctx->sd->control_net != nullptr) {
+ return true;
+ }
+ // z-image keeps control net weights inside the diffusion runner, registered
+ // under ModelComponent::ControlNet instead of a separate control_net runner.
+ return sd_ctx->sd->model_manager != nullptr &&
+ sd_ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0;
+}
+
+const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx) {
+ if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_ctx->sd->version >= VERSION_COUNT) {
+ return "Unknown";
+ }
+ return sd_version_name(sd_ctx->sd->version);
+}
+
+sd_status_t sd_generate_image(sd_ctx_t* ctx, const sd_img_gen_opt_t* params, sd_result_t** result_out) {
+ if (result_out)
+ *result_out = nullptr;
+ return sd::api::call([&] {
+ if (!result_out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result_out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(params, sd::api::Image, config);
+ const auto& image = config.image;
+ if (status != SD_STATUS_OK)
+ return status;
+ sd::ImageGenerationResult* images = nullptr;
+ status = sd::generate_image(ctx, image, &images, params);
+ if (status != SD_STATUS_OK)
+ return status;
+ return sd::make_image_result(images, result_out) ? SD_STATUS_OK
+ : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result");
+ },
+ SD_STATUS_GENERATION_FAILED);
+}
+
+sd_status_t sd_generate_video(sd_ctx_t* ctx, const sd_vid_gen_opt_t* params, sd_result_t** result_out) {
+ if (result_out)
+ *result_out = nullptr;
+ return sd::api::call([&] {
+ if (!result_out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result_out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(params, sd::api::Video, config);
+ const auto& video = config.video;
+ if (status != SD_STATUS_OK)
+ return status;
+ sd::VideoGenerationResult* frames = nullptr;
+ status = sd::generate_video(ctx, video, &frames, params);
+ if (status != SD_STATUS_OK)
+ return status;
+ return sd::make_video_result(frames, result_out) ? SD_STATUS_OK
+ : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result");
+ },
+ SD_STATUS_GENERATION_FAILED);
+}
+
+sd_status_t sd_preprocess_canny(const sd_tensor_t* image, float high_threshold, float low_threshold, float weak, float strong, bool inverse, sd_tensor_t** image_out) {
+ if (image_out)
+ *image_out = nullptr;
+ return sd::api::call([&] {
+ if (!image || !image_out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "image and image_out are required");
+ sd::api::image_view(*image);
+ auto output = sd::api::allocate_tensor(image->type, image->shape);
+ memcpy(output.get()->mutable_data(), image->data(), image->bytes);
+ if (!sd::preprocess_canny(sd::api::image_view(*output.get()), high_threshold, low_threshold, weak, strong, inverse))
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "Canny preprocessing failed");
+ *image_out = sd_tensor_retain(output.get());
+ return SD_STATUS_OK;
+ });
+}
diff --git a/src/api/api.h b/src/api/api.h
new file mode 100644
index 0000000..545859e
--- /dev/null
+++ b/src/api/api.h
@@ -0,0 +1,34 @@
+#ifndef SD_INTERNAL_API_H
+#define SD_INTERNAL_API_H
+
+#include <exception>
+#include <new>
+
+#include "stable-diffusion.h"
+
+class StableDiffusionGGML;
+
+struct sd_ctx_t {
+ StableDiffusionGGML* sd = nullptr;
+};
+
+namespace sd::api {
+ void clear_error();
+ sd_status_t error(sd_status_t status, const char* key, const char* reason);
+
+ template <typename Function>
+ sd_status_t call(Function&& function, sd_status_t failure = SD_STATUS_INVALID_ARGUMENT) noexcept {
+ clear_error();
+ try {
+ return function();
+ } catch (const std::bad_alloc&) {
+ return error(SD_STATUS_OUT_OF_MEMORY, nullptr, "out of memory");
+ } catch (const std::exception& exception) {
+ return error(failure, nullptr, exception.what());
+ } catch (...) {
+ return error(failure, nullptr, "unexpected exception");
+ }
+ }
+}
+
+#endif // SD_INTERNAL_API_H
diff --git a/src/api/options.cpp b/src/api/options.cpp
new file mode 100644
index 0000000..591cce3
--- /dev/null
+++ b/src/api/options.cpp
@@ -0,0 +1,897 @@
+#include "options.h"
+#include "core/option_types.h"
+#include "tensor.h"
+
+#include <algorithm>
+#include <cmath>
+#include <functional>
+#include <limits>
+#include <map>
+#include <new>
+#include <stdexcept>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+
+enum class ParamType { Bool,
+ Int,
+ Float,
+ String,
+ Tensor,
+ Tensors,
+ Floats,
+ Ints,
+ Loras,
+ Embeddings,
+ Audios,
+ Videos };
+
+struct LoraValues {
+ std::vector<std::string> paths;
+ std::vector<sd_lora_t> items;
+};
+
+struct EmbeddingValues {
+ std::vector<std::string> strings;
+ std::vector<sd_embedding_t> items;
+};
+
+struct TensorValue {
+ sd::api::TensorRef tensor;
+ std::vector<sd::Image> frames;
+};
+
+struct ImageValues {
+ std::vector<sd::api::TensorRef> tensors;
+ std::vector<sd::Image> items;
+};
+
+struct AudioValues {
+ std::vector<sd::api::TensorRef> tensors;
+ std::vector<sd::Audio> items;
+};
+
+struct VideoValues {
+ std::vector<sd::api::TensorRef> tensors;
+ std::vector<std::vector<sd::Image>> frames;
+ std::vector<sd::ReferenceVideo> items;
+};
+
+using ParamValue = std::variant<bool, int64_t, double, std::string, TensorValue, ImageValues, std::vector<float>, std::vector<int>, LoraValues, EmbeddingValues, AudioValues, VideoValues>;
+
+struct sd_opt {
+ sd::api::Operation operation;
+ std::map<std::string, ParamValue> values;
+};
+
+struct ParamSpec {
+ const char* key;
+ ParamType type;
+ int operations;
+ std::function<void(const ParamValue&, sd::OptConfig&)> apply;
+ int64_t int_min = std::numeric_limits<int64_t>::min();
+ int64_t int_max = std::numeric_limits<int64_t>::max();
+ double float_min = -std::numeric_limits<float>::max();
+ double float_max = std::numeric_limits<float>::max();
+ std::function<bool(const ParamValue&)> valid;
+ const char* invalid_reason = "invalid parameter value";
+ bool video_tensor = false;
+};
+
+static sd_cache_mode_t parse_cache(const char* text) {
+ static const char* names[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"};
+ for (int i = 0; i < 7; ++i)
+ if (strcmp(text, names[i]) == 0)
+ return static_cast<sd_cache_mode_t>(i);
+ return static_cast<sd_cache_mode_t>(-1);
+}
+
+static sd_vae_format_t parse_vae_format(const char* text) {
+ static const char* names[] = {"auto", "flux", "sd3", "flux2", "wan"};
+ for (int i = 0; i < 5; ++i)
+ if (strcmp(text, names[i]) == 0)
+ return static_cast<sd_vae_format_t>(i - 1);
+ return SD_VAE_FORMAT_COUNT;
+}
+
+static sd_hires_upscaler_t parse_hires_upscaler(const char* text) {
+ static const char* names[] = {"none", "latent", "latent_nearest", "latent_nearest_exact",
+ "latent_antialiased", "latent_bicubic", "latent_bicubic_antialiased",
+ "lanczos", "nearest", "model"};
+ for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; ++i)
+ if (strcmp(text, names[i]) == 0)
+ return static_cast<sd_hires_upscaler_t>(i);
+ return SD_HIRES_UPSCALER_COUNT;
+}
+
+static bool valid_sigmas(const ParamValue& value) {
+ const auto& sigmas = std::get<std::vector<float>>(value);
+ if (sigmas.empty())
+ return true;
+ if (sigmas.size() < 2 || sigmas.back() != 0.f)
+ return false;
+ for (size_t i = 0; i < sigmas.size(); ++i)
+ if (sigmas[i] < 0.f || (i && sigmas[i] >= sigmas[i - 1]))
+ return false;
+ return true;
+}
+
+static bool valid_layers(const ParamValue& value) {
+ const auto& layers = std::get<std::vector<int>>(value);
+ return std::all_of(layers.begin(), layers.end(), [](int index) { return index >= 0; });
+}
+
+template <typename Value, typename Apply>
+static ParamSpec parameter(const char* key, ParamType type, int operations, Apply apply) {
+ return {key, type, operations, [apply](const ParamValue& value, sd::OptConfig& config) {
+ apply(std::get<Value>(value), config);
+ }};
+}
+
+template <typename Apply>
+static ParamSpec integer_parameter(const char* key, int operations, Apply apply, int64_t min = INT64_MIN, int64_t max = INT64_MAX) {
+ auto spec = parameter<int64_t>(key, ParamType::Int, operations, apply);
+ spec.int_min = min;
+ spec.int_max = max;
+ return spec;
+}
+
+template <typename Apply>
+static ParamSpec float_parameter(const char* key, int operations, Apply apply, double min = -std::numeric_limits<float>::max(), double max = std::numeric_limits<float>::max()) {
+ auto spec = parameter<double>(key, ParamType::Float, operations, apply);
+ spec.float_min = min;
+ spec.float_max = max;
+ return spec;
+}
+
+template <typename T>
+struct ArrayStorage;
+template <>
+struct ArrayStorage<float> {
+ using Value = std::vector<float>;
+ static constexpr auto type = ParamType::Floats;
+};
+template <>
+struct ArrayStorage<int> {
+ using Value = std::vector<int>;
+ static constexpr auto type = ParamType::Ints;
+};
+template <>
+struct ArrayStorage<sd::Image> {
+ using Value = ImageValues;
+ static constexpr auto type = ParamType::Tensors;
+};
+template <>
+struct ArrayStorage<sd::Audio> {
+ using Value = AudioValues;
+ static constexpr auto type = ParamType::Audios;
+};
+template <>
+struct ArrayStorage<sd::ReferenceVideo> {
+ using Value = VideoValues;
+ static constexpr auto type = ParamType::Videos;
+};
+template <>
+struct ArrayStorage<sd_lora_t> {
+ using Value = LoraValues;
+ static constexpr auto type = ParamType::Loras;
+};
+template <>
+struct ArrayStorage<sd_embedding_t> {
+ using Value = EmbeddingValues;
+ static constexpr auto type = ParamType::Embeddings;
+};
+
+template <typename T>
+static const auto& array_items(const std::vector<T>& value) {
+ return value;
+}
+static const auto& array_items(const ImageValues& value) {
+ return value.items;
+}
+static const auto& array_items(const AudioValues& value) {
+ return value.items;
+}
+static const auto& array_items(const VideoValues& value) {
+ return value.items;
+}
+static const auto& array_items(const LoraValues& value) {
+ return value.items;
+}
+static const auto& array_items(const EmbeddingValues& value) {
+ return value.items;
+}
+
+template <typename Select>
+class ParamGroup {
+ std::vector<ParamSpec>& specs;
+ int operations;
+ Select select;
+
+ template <typename Value, typename Object, typename Field, typename Convert>
+ ParamSpec binding(const char* key, ParamType type, Field Object::* field, Convert convert) const {
+ return parameter<Value>(key, type, operations, [select = select, field, convert](const Value& value, sd::OptConfig& config) {
+ const auto decoded = convert(value);
+ std::apply([&](auto&... target) { ((target.*field = decoded), ...); }, select(config));
+ });
+ }
+
+public:
+ ParamGroup(std::vector<ParamSpec>& specs, int operations, Select select)
+ : specs(specs), operations(operations), select(select) {}
+
+ ParamGroup for_operations(int mask) const { return {specs, mask, select}; }
+
+ template <typename Object>
+ void boolean(const char* key, bool Object::* field) {
+ specs.push_back(binding<bool>(key, ParamType::Bool, field, [](bool value) { return value; }));
+ }
+
+ template <typename Object, typename Field>
+ void integer(const char* key, Field Object::* field, int64_t min = INT64_MIN, int64_t max = INT64_MAX) {
+ auto spec = binding<int64_t>(key, ParamType::Int, field, [](int64_t value) { return static_cast<Field>(value); });
+ spec.int_min = min;
+ spec.int_max = max;
+ specs.push_back(std::move(spec));
+ }
+
+ template <typename Object>
+ void number(const char* key, float Object::* field, double min = -std::numeric_limits<float>::max(), double max = std::numeric_limits<float>::max()) {
+ auto spec = binding<double>(key, ParamType::Float, field, [](double value) { return static_cast<float>(value); });
+ spec.float_min = min;
+ spec.float_max = max;
+ specs.push_back(std::move(spec));
+ }
+
+ template <typename Object>
+ void string(const char* key, const char* Object::* field) {
+ specs.push_back(binding<std::string>(key, ParamType::String, field, [](const std::string& value) { return value.c_str(); }));
+ }
+
+ template <typename Object>
+ void image(const char* key, sd::Image Object::* field) {
+ specs.push_back(binding<TensorValue>(key, ParamType::Tensor, field, [](const TensorValue& value) { return sd::api::image_view(*value.tensor.get()); }));
+ }
+
+ template <typename Object, typename Enum>
+ void choice(const char* key, Enum Object::* field, Enum (*parse)(const char*), Enum invalid, bool allow_auto = false) {
+ auto spec = binding<std::string>(key, ParamType::String, field, [=](const std::string& value) {
+ return allow_auto && value == "auto" ? invalid : parse(value.c_str());
+ });
+ spec.valid = [=](const ParamValue& value) {
+ const auto& text = std::get<std::string>(value);
+ return (allow_auto && text == "auto") || parse(text.c_str()) != invalid;
+ };
+ specs.push_back(std::move(spec));
+ }
+
+ template <typename Object, typename Element, typename Count>
+ void frames(const char* key, Element* Object::* data, Count Object::* count) {
+ auto spec = parameter<TensorValue>(key, ParamType::Tensor, operations, [select = select, data, count](const TensorValue& value, sd::OptConfig& config) {
+ std::apply([&](auto&... target) {
+ ((target.*data = const_cast<Element*>(value.frames.data()), target.*count = static_cast<Count>(value.frames.size())), ...);
+ },
+ select(config));
+ });
+ spec.video_tensor = true;
+ specs.push_back(std::move(spec));
+ }
+
+ template <typename Object, typename Element, typename Count>
+ void array(const char* key, Element* Object::* data, Count Object::* count, bool (*valid)(const ParamValue&) = nullptr, const char* reason = "invalid parameter value") {
+ using Storage = ArrayStorage<std::remove_const_t<Element>>;
+ using Value = typename Storage::Value;
+ auto spec = parameter<Value>(key, Storage::type, operations, [select = select, data, count](const Value& value, sd::OptConfig& config) {
+ const auto& items = array_items(value);
+ std::apply([&](auto&... target) {
+ ((target.*data = const_cast<Element*>(items.data()), target.*count = static_cast<Count>(items.size())), ...);
+ },
+ select(config));
+ });
+ spec.valid = valid;
+ spec.invalid_reason = reason;
+ specs.push_back(std::move(spec));
+ }
+};
+
+static void add_context_specs(std::vector<ParamSpec>& specs) {
+ using namespace sd::api;
+ ParamGroup ctx(specs, Context, [](sd::OptConfig& config) { return std::tie(config.context); });
+
+ ctx.number("attn_scale", &sd::ContextConfig::attn_scale, 0, std::numeric_limits<float>::max());
+ ctx.string("audio_encoder_path", &sd::ContextConfig::audio_encoder_path);
+ ctx.string("audio_vae_path", &sd::ContextConfig::audio_vae_path);
+ ctx.boolean("auto_fit", &sd::ContextConfig::auto_fit);
+ ctx.boolean("batched_cfg", &sd::ContextConfig::batched_cfg);
+ ctx.string("clip_g_path", &sd::ContextConfig::clip_g_path);
+ ctx.string("clip_l_path", &sd::ContextConfig::clip_l_path);
+ ctx.string("clip_vision_path", &sd::ContextConfig::clip_vision_path);
+ ctx.integer("conditioning_cache_size", &sd::ContextConfig::conditioning_cache_size, 0, INT32_MAX);
+ ctx.string("control_net_path", &sd::ContextConfig::control_net_path);
+ ctx.boolean("diffusion_conv_direct", &sd::ContextConfig::diffusion_conv_direct);
+ ctx.boolean("diffusion_flash_attn", &sd::ContextConfig::diffusion_flash_attn);
+ ctx.boolean("disable_prefetch", &sd::ContextConfig::disable_prefetch);
+ ctx.boolean("disable_segmented_compute", &sd::ContextConfig::disable_segmented_compute);
+ ctx.boolean("eager_load", &sd::ContextConfig::eager_load);
+ ctx.array("embeddings", &sd::ContextConfig::embeddings, &sd::ContextConfig::embedding_count);
+ ctx.string("embeddings_connectors_path", &sd::ContextConfig::embeddings_connectors_path);
+ ctx.boolean("enable_mmap", &sd::ContextConfig::enable_mmap);
+ ctx.boolean("flash_attn", &sd::ContextConfig::flash_attn);
+ ctx.boolean("force_sdxl_vae_conv_scale", &sd::ContextConfig::force_sdxl_vae_conv_scale);
+ ctx.string("high_noise_diffusion_model_path", &sd::ContextConfig::high_noise_diffusion_model_path);
+ ctx.string("ip_adapter_path", &sd::ContextConfig::ip_adapter_path);
+ ctx.number("linear_scale", &sd::ContextConfig::linear_scale, 0, std::numeric_limits<float>::max());
+ ctx.string("llm_path", &sd::ContextConfig::llm_path);
+ ctx.string("llm_vision_path", &sd::ContextConfig::llm_vision_path);
+ ctx.choice("lora_apply_mode", &sd::ContextConfig::lora_apply_mode, str_to_lora_apply_mode, LORA_APPLY_MODE_COUNT);
+ ctx.string("max_vram", &sd::ContextConfig::max_vram);
+ ctx.string("model_args", &sd::ContextConfig::model_args);
+ ctx.for_operations(Context | Upscaler | ADetailer).string("model_path", &sd::ContextConfig::model_path);
+ ctx.string("motion_module_path", &sd::ContextConfig::motion_module_path);
+ ctx.string("photo_maker_path", &sd::ContextConfig::photo_maker_path);
+ ctx.choice("prediction", &sd::ContextConfig::prediction, str_to_prediction, PREDICTION_COUNT, true);
+ ctx.string("pulid_weights_path", &sd::ContextConfig::pulid_weights_path);
+ ctx.choice("rng_type", &sd::ContextConfig::rng_type, str_to_rng_type, RNG_TYPE_COUNT, true);
+ ctx.string("rpc_servers", &sd::ContextConfig::rpc_servers);
+ ctx.boolean("sage_attn", &sd::ContextConfig::sage_attn);
+ ctx.choice("sampler_rng_type", &sd::ContextConfig::sampler_rng_type, str_to_rng_type, RNG_TYPE_COUNT, true);
+ ctx.string("split_mode", &sd::ContextConfig::split_mode);
+ ctx.string("t5xxl_path", &sd::ContextConfig::t5xxl_path);
+ ctx.boolean("tae_preview_only", &sd::ContextConfig::tae_preview_only);
+ ctx.string("taesd_path", &sd::ContextConfig::taesd_path);
+ ctx.string("tensor_type_rules", &sd::ContextConfig::tensor_type_rules);
+ ctx.string("tokenizer", &sd::ContextConfig::tokenizer);
+ ctx.string("uncond_diffusion_model_path", &sd::ContextConfig::uncond_diffusion_model_path);
+ ctx.boolean("vae_conv_direct", &sd::ContextConfig::vae_conv_direct);
+ ctx.choice("vae_format", &sd::ContextConfig::vae_format, parse_vae_format, SD_VAE_FORMAT_COUNT);
+ ctx.choice("wtype", &sd::ContextConfig::wtype, str_to_sd_type, SD_TYPE_COUNT, true);
+
+ specs.push_back(parameter<std::string>("backend", ParamType::String, Context | Upscaler | VidUpscaler | ADetailer, [](const std::string& value, sd::OptConfig& config) { config.context.backend = value.c_str(); config.post.video_upscaler.backend = value.c_str(); }));
+ specs.push_back(parameter<std::string>("conv_direct_mode", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.conv_direct_mode = value.c_str(); config.post.video_upscaler.conv_direct_mode = value.c_str(); }));
+ specs.push_back(parameter<std::string>("diffusion_model_path", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.diffusion_model_path = value.c_str(); config.post.video_upscaler.diffusion_model_path = value.c_str(); }));
+ specs.push_back(parameter<std::string>("flash_attn_mode", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.flash_attn_mode = value.c_str(); config.post.video_upscaler.flash_attn_mode = value.c_str(); }));
+ specs.push_back(integer_parameter(
+ "n_threads", Context | Upscaler | VidUpscaler | ADetailer, [](const int64_t& value, sd::OptConfig& config) { config.context.n_threads = static_cast<int>(value); config.post.video_upscaler.n_threads = static_cast<int>(value); }, 1, INT32_MAX));
+ specs.push_back(parameter<std::string>("params_backend", ParamType::String, Context | Upscaler | VidUpscaler | ADetailer, [](const std::string& value, sd::OptConfig& config) { config.context.params_backend = value.c_str(); config.post.video_upscaler.params_backend = value.c_str(); }));
+ specs.push_back(parameter<std::string>("vae_path", ParamType::String, Context | VidUpscaler, [](const std::string& value, sd::OptConfig& config) { config.context.vae_path = value.c_str(); config.post.video_upscaler.vae_path = value.c_str(); }));
+}
+
+static void add_generation_specs(std::vector<ParamSpec>& specs) {
+ using namespace sd::api;
+ ParamGroup img(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image); });
+ ParamGroup vid(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video); });
+ ParamGroup preprocess(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.image_preprocess, config.video.image_preprocess); });
+ ParamGroup photomaker(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image.pm_params); });
+ ParamGroup pulid(specs, Image, [](sd::OptConfig& config) { return std::tie(config.image.pulid_params); });
+
+ img.integer("batch_count", &sd::ImageConfig::batch_count, 1, INT32_MAX);
+ img.image("control_image", &sd::ImageConfig::control_image);
+ img.number("control_strength", &sd::ImageConfig::control_strength, 0, std::numeric_limits<float>::max());
+ img.choice("control_type", &sd::ImageConfig::control_type, str_to_sd_control_type, SD_CONTROL_TYPE_COUNT);
+ img.image("ip_adapter_image", &sd::ImageConfig::ip_adapter_image);
+ img.number("ip_adapter_strength", &sd::ImageConfig::ip_adapter_strength, 0, std::numeric_limits<float>::max());
+ img.image("mask_image", &sd::ImageConfig::mask_image);
+ img.integer("qwen_image_layers", &sd::ImageConfig::qwen_image_layers, 0, INT32_MAX);
+
+ vid.frames("control_frames", &sd::VideoConfig::control_frames, &sd::VideoConfig::control_frames_size);
+ vid.image("end_image", &sd::VideoConfig::end_image);
+ vid.number("moe_boundary", &sd::VideoConfig::moe_boundary, 0, 1);
+ vid.array("ref_audios", &sd::VideoConfig::ref_audios, &sd::VideoConfig::ref_audios_count);
+ vid.array("ref_videos", &sd::VideoConfig::ref_videos, &sd::VideoConfig::ref_videos_count);
+ vid.number("vace_strength", &sd::VideoConfig::vace_strength, 0, std::numeric_limits<float>::max());
+ vid.integer("video_frames", &sd::VideoConfig::video_frames, 1, INT32_MAX);
+
+ preprocess.string("image_preprocess.rules", &sd::ImagePreprocessConfig::rules);
+
+ photomaker.string("photomaker.id_embed_path", &sd::PhotoMakerConfig::id_embed_path);
+ photomaker.array("photomaker.id_images", &sd::PhotoMakerConfig::id_images, &sd::PhotoMakerConfig::id_images_count);
+ photomaker.number("photomaker.style_strength", &sd::PhotoMakerConfig::style_strength, 0, 100);
+
+ pulid.string("pulid.id_embedding_path", &sd::PuLIDConfig::id_embedding_path);
+ pulid.number("pulid.id_weight", &sd::PuLIDConfig::id_weight, 0, std::numeric_limits<float>::max());
+
+ specs.push_back(parameter<bool>("circular_x", ParamType::Bool, Image | Video | ADetail, [](const bool& value, sd::OptConfig& config) { config.image.circular_x = value; config.video.circular_x = value; }));
+ specs.push_back(parameter<bool>("circular_y", ParamType::Bool, Image | Video | ADetail, [](const bool& value, sd::OptConfig& config) { config.image.circular_y = value; config.video.circular_y = value; }));
+ specs.push_back(integer_parameter(
+ "clip_skip", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.clip_skip = static_cast<int>(value); config.video.clip_skip = static_cast<int>(value); }, INT32_MIN, INT32_MAX));
+ specs.push_back(integer_parameter(
+ "fps", Video | VidUpscale, [](const int64_t& value, sd::OptConfig& config) { config.video.fps = static_cast<int>(value); config.post.video_upscale.fps = static_cast<int>(value); }, 1, INT32_MAX));
+ specs.push_back(integer_parameter(
+ "height", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.height = static_cast<int>(value); config.video.height = static_cast<int>(value); }, 1, INT32_MAX));
+ specs.push_back(parameter<TensorValue>("init_image", ParamType::Tensor, Image | Video, [](const TensorValue& value, sd::OptConfig& config) { config.image.init_image = sd::api::image_view(*value.tensor.get()); config.video.init_image = config.image.init_image; }));
+ specs.push_back(parameter<LoraValues>("loras", ParamType::Loras, Image | Video | ADetail, [](const LoraValues& value, sd::OptConfig& config) { config.image.loras = value.items.data(); config.image.lora_count = static_cast<decltype(config.image.lora_count)>(value.items.size()); config.video.loras = value.items.data(); config.video.lora_count = static_cast<decltype(config.video.lora_count)>(value.items.size()); }));
+ specs.push_back(parameter<std::string>("negative_prompt", ParamType::String, Image | Video | ADetail, [](const std::string& value, sd::OptConfig& config) { config.image.negative_prompt = value.c_str(); config.video.negative_prompt = value.c_str(); }));
+ specs.push_back(parameter<std::string>("prompt", ParamType::String, Image | Video | ADetail, [](const std::string& value, sd::OptConfig& config) { config.image.prompt = value.c_str(); config.video.prompt = value.c_str(); }));
+ specs.push_back(parameter<std::string>("ref_image_args", ParamType::String, Image | Video, [](const std::string& value, sd::OptConfig& config) { config.image.ref_image_args = value.c_str(); config.video.ref_image_args = value.c_str(); }));
+ specs.push_back(parameter<ImageValues>("ref_images", ParamType::Tensors, Image | Video, [](const ImageValues& value, sd::OptConfig& config) { config.image.ref_images = const_cast<sd::Image*>(value.items.data()); config.image.ref_images_count = static_cast<int>(value.items.size()); config.video.ref_images = config.image.ref_images; config.video.ref_images_count = config.image.ref_images_count; }));
+ specs.push_back(integer_parameter("seed", Image | Video | VidUpscale | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.seed = static_cast<int64_t>(value); config.video.seed = static_cast<int64_t>(value); config.post.video_upscale.seed = value; }));
+ specs.push_back(float_parameter(
+ "strength", Image | Video | ADetail, [](const double& value, sd::OptConfig& config) { config.image.strength = static_cast<float>(value); config.video.strength = static_cast<float>(value); }, 0, 1));
+ specs.push_back(integer_parameter(
+ "width", Image | Video | ADetail, [](const int64_t& value, sd::OptConfig& config) { config.image.width = static_cast<int>(value); config.video.width = static_cast<int>(value); }, 1, INT32_MAX));
+}
+
+static void add_sampling_specs(std::vector<ParamSpec>& specs) {
+ using namespace sd::api;
+ ParamGroup high_noise(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params); });
+ ParamGroup high_noise_guidance(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params.guidance); });
+ ParamGroup high_noise_slg(specs, Video, [](sd::OptConfig& config) { return std::tie(config.video.high_noise_sample_params.guidance.slg); });
+ ParamGroup sample(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params, config.video.sample_params); });
+ ParamGroup guidance(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance, config.video.sample_params.guidance); });
+ ParamGroup slg(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.sample_params.guidance.slg, config.video.sample_params.guidance.slg); });
+
+ high_noise.string("high_noise_sample.args", &sd::SampleConfig::extra_sample_args);
+ high_noise.number("high_noise_sample.eta", &sd::SampleConfig::eta, 0, std::numeric_limits<float>::max());
+ high_noise.choice("high_noise_sample.method", &sd::SampleConfig::sample_method, str_to_sample_method, SAMPLE_METHOD_COUNT, true);
+ high_noise.integer("high_noise_sample.steps", &sd::SampleConfig::sample_steps, -1, INT32_MAX);
+
+ high_noise_guidance.number("high_noise_sample.guidance.distilled_guidance", &sd::GuidanceConfig::distilled_guidance);
+ high_noise_guidance.number("high_noise_sample.guidance.img_cfg", &sd::GuidanceConfig::img_cfg);
+ high_noise_guidance.number("high_noise_sample.guidance.txt_cfg", &sd::GuidanceConfig::txt_cfg);
+
+ high_noise_slg.number("high_noise_sample.guidance.slg.layer_end", &sd::SLGConfig::layer_end, 0, 1);
+ high_noise_slg.number("high_noise_sample.guidance.slg.layer_start", &sd::SLGConfig::layer_start, 0, 1);
+ high_noise_slg.array("high_noise_sample.guidance.slg.layers", &sd::SLGConfig::layers, &sd::SLGConfig::layer_count, valid_layers, "layer indices must be nonnegative");
+ high_noise_slg.number("high_noise_sample.guidance.slg.scale", &sd::SLGConfig::scale, 0, std::numeric_limits<float>::max());
+
+ sample.string("sample.args", &sd::SampleConfig::extra_sample_args);
+ sample.number("sample.eta", &sd::SampleConfig::eta, 0, std::numeric_limits<float>::max());
+ sample.number("sample.flow_shift", &sd::SampleConfig::flow_shift, 0, std::numeric_limits<float>::max());
+ sample.choice("sample.method", &sd::SampleConfig::sample_method, str_to_sample_method, SAMPLE_METHOD_COUNT, true);
+ sample.choice("sample.scheduler", &sd::SampleConfig::scheduler, str_to_scheduler, SCHEDULER_COUNT, true);
+ sample.integer("sample.shifted_timestep", &sd::SampleConfig::shifted_timestep, 0, INT32_MAX);
+ sample.array("sample.sigmas", &sd::SampleConfig::custom_sigmas, &sd::SampleConfig::custom_sigmas_count, valid_sigmas, "sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array");
+ sample.integer("sample.steps", &sd::SampleConfig::sample_steps, 1, INT32_MAX);
+
+ guidance.number("sample.guidance.distilled_guidance", &sd::GuidanceConfig::distilled_guidance);
+ guidance.number("sample.guidance.img_cfg", &sd::GuidanceConfig::img_cfg);
+ guidance.number("sample.guidance.txt_cfg", &sd::GuidanceConfig::txt_cfg);
+
+ slg.number("sample.guidance.slg.layer_end", &sd::SLGConfig::layer_end, 0, 1);
+ slg.number("sample.guidance.slg.layer_start", &sd::SLGConfig::layer_start, 0, 1);
+ slg.array("sample.guidance.slg.layers", &sd::SLGConfig::layers, &sd::SLGConfig::layer_count, valid_layers, "layer indices must be nonnegative");
+ slg.number("sample.guidance.slg.scale", &sd::SLGConfig::scale, 0, std::numeric_limits<float>::max());
+}
+
+static void add_cache_specs(std::vector<ParamSpec>& specs) {
+ using namespace sd::api;
+ ParamGroup cache(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.cache, config.video.cache); });
+
+ cache.integer("cache.bn_compute_blocks", &sd::CacheConfig::Bn_compute_blocks, 0, INT32_MAX);
+ cache.number("cache.end_percent", &sd::CacheConfig::end_percent, 0, 1);
+ cache.number("cache.error_decay_rate", &sd::CacheConfig::error_decay_rate, 0, 1);
+ cache.integer("cache.fn_compute_blocks", &sd::CacheConfig::Fn_compute_blocks, 0, INT32_MAX);
+ cache.integer("cache.max_cached_steps", &sd::CacheConfig::max_cached_steps, INT32_MIN, INT32_MAX);
+ cache.integer("cache.max_continuous_cached_steps", &sd::CacheConfig::max_continuous_cached_steps, INT32_MIN, INT32_MAX);
+ cache.integer("cache.max_warmup_steps", &sd::CacheConfig::max_warmup_steps, 0, INT32_MAX);
+ cache.choice("cache.mode", &sd::CacheConfig::mode, parse_cache, static_cast<sd_cache_mode_t>(-1));
+ cache.boolean("cache.reset_error_on_compute", &sd::CacheConfig::reset_error_on_compute);
+ cache.number("cache.residual_diff_threshold", &sd::CacheConfig::residual_diff_threshold, 0, std::numeric_limits<float>::max());
+ cache.number("cache.reuse_threshold", &sd::CacheConfig::reuse_threshold, 0, std::numeric_limits<float>::max());
+ cache.string("cache.scm_mask", &sd::CacheConfig::scm_mask);
+ cache.boolean("cache.scm_policy_dynamic", &sd::CacheConfig::scm_policy_dynamic);
+ cache.number("cache.spectrum_flex_window", &sd::CacheConfig::spectrum_flex_window, 0, std::numeric_limits<float>::max());
+ cache.number("cache.spectrum_lam", &sd::CacheConfig::spectrum_lam, 0, std::numeric_limits<float>::max());
+ cache.integer("cache.spectrum_m", &sd::CacheConfig::spectrum_m, 1, INT32_MAX);
+ cache.number("cache.spectrum_stop_percent", &sd::CacheConfig::spectrum_stop_percent, 0, 1);
+ cache.number("cache.spectrum_w", &sd::CacheConfig::spectrum_w, 0, std::numeric_limits<float>::max());
+ cache.integer("cache.spectrum_warmup_steps", &sd::CacheConfig::spectrum_warmup_steps, 0, INT32_MAX);
+ cache.integer("cache.spectrum_window_size", &sd::CacheConfig::spectrum_window_size, 1, INT32_MAX);
+ cache.number("cache.start_percent", &sd::CacheConfig::start_percent, 0, 1);
+ cache.integer("cache.taylorseer_n_derivatives", &sd::CacheConfig::taylorseer_n_derivatives, 0, INT32_MAX);
+ cache.integer("cache.taylorseer_skip_interval", &sd::CacheConfig::taylorseer_skip_interval, 1, INT32_MAX);
+ cache.boolean("cache.use_relative_threshold", &sd::CacheConfig::use_relative_threshold);
+}
+
+static void add_hires_specs(std::vector<ParamSpec>& specs) {
+ using namespace sd::api;
+ ParamGroup hires(specs, Image | Video, [](sd::OptConfig& config) { return std::tie(config.image.hires, config.video.hires); });
+
+ hires.number("hires.denoising_strength", &sd::HiresConfig::denoising_strength, 0, 1);
+ hires.boolean("hires.enabled", &sd::HiresConfig::enabled);
+ hires.string("hires.model_path", &sd::HiresConfig::model_path);
+ hires.number("hires.scale", &sd::HiresConfig::scale, 0, std::numeric_limits<float>::max());
+ hires.array("hires.sigmas", &sd::HiresConfig::custom_sigmas, &sd::HiresConfig::custom_sigmas_count, valid_sigmas, "sigmas require at least two strictly decreasing nonnegative entries ending in zero, or an empty array");
+ hires.integer("hires.steps", &sd::HiresConfig::steps, 0, INT32_MAX);
+ hires.integer("hires.target_height", &sd::HiresConfig::target_height, 0, INT32_MAX);
+ hires.integer("hires.target_width", &sd::HiresConfig::target_width, 0, INT32_MAX);
+ hires.integer("hires.upscale_tile_size", &sd::HiresConfig::upscale_tile_size, 1, INT32_MAX);
+ hires.choice("hires.upscaler", &sd::HiresConfig::upscaler, parse_hires_upscaler, SD_HIRES_UPSCALER_COUNT);
+}
+
+static void add_tiling_specs(std::vector<ParamSpec>& specs) {
+ using namespace sd::api;
+ ParamGroup tiling(specs, Image | Video | ADetail, [](sd::OptConfig& config) { return std::tie(config.image.vae_tiling_params, config.video.vae_tiling_params); });
+
+ tiling.boolean("vae_tiling.enabled", &sd::TilingConfig::enabled);
+ tiling.string("vae_tiling.extra_tiling_args", &sd::TilingConfig::extra_tiling_args);
+ tiling.number("vae_tiling.rel_size_h", &sd::TilingConfig::rel_size_h, 0, std::numeric_limits<float>::max());
+ tiling.number("vae_tiling.rel_size_w", &sd::TilingConfig::rel_size_w, 0, std::numeric_limits<float>::max());
+ tiling.number("vae_tiling.target_overlap", &sd::TilingConfig::target_overlap, 0, 0.99);
+ tiling.boolean("vae_tiling.temporal_tiling", &sd::TilingConfig::temporal_tiling);
+ tiling.integer("vae_tiling.tile_size_h", &sd::TilingConfig::tile_size_h, 0, INT32_MAX);
+ tiling.integer("vae_tiling.tile_size_w", &sd::TilingConfig::tile_size_w, 0, INT32_MAX);
+}
+
+static void add_postprocess_specs(std::vector<ParamSpec>& specs) {
+ using namespace sd::api;
+ ParamGroup input(specs, Upscale | ADetail, [](sd::OptConfig& config) { return std::tie(config.post); });
+ ParamGroup upscaler(specs, Upscaler, [](sd::OptConfig& config) { return std::tie(config.post); });
+ ParamGroup video_upscaler(specs, VidUpscaler, [](sd::OptConfig& config) { return std::tie(config.post.video_upscaler); });
+ ParamGroup video_upscale(specs, VidUpscale, [](sd::OptConfig& config) { return std::tie(config.post.video_upscale); });
+ ParamGroup adetail(specs, ADetail, [](sd::OptConfig& config) { return std::tie(config.post.adetail); });
+
+ input.image("image", &sd::PostprocessConfig::image);
+
+ upscaler.boolean("conv_direct", &sd::PostprocessConfig::conv_direct);
+ upscaler.integer("tile_size", &sd::PostprocessConfig::tile_size, 1, INT32_MAX);
+
+ video_upscaler.integer("upscale_factor", &sd::VideoUpscalerConfig::upscale_factor, 1, INT32_MAX);
+
+ video_upscale.frames("frames", &sd::VideoUpscaleConfig::frames, &sd::VideoUpscaleConfig::frame_count);
+ video_upscale.integer("window_size", &sd::VideoUpscaleConfig::window_size, 1, INT32_MAX);
+ video_upscale.integer("temporal_chunk_size", &sd::VideoUpscaleConfig::temporal_chunk_size, 1, INT32_MAX);
+
+ adetail.string("adetail.prompt", &sd::ADetailConfig::prompt);
+ adetail.string("adetail.negative_prompt", &sd::ADetailConfig::negative_prompt);
+ adetail.string("adetail.args", &sd::ADetailConfig::extra_ad_args);
+}
+
+static std::vector<ParamSpec> make_specs() {
+ std::vector<ParamSpec> specs;
+ add_context_specs(specs);
+ add_generation_specs(specs);
+ add_sampling_specs(specs);
+ add_cache_specs(specs);
+ add_hires_specs(specs);
+ add_tiling_specs(specs);
+ add_postprocess_specs(specs);
+ return specs;
+}
+
+static const ParamSpec* find_spec(const char* key) {
+ static const auto specs = make_specs();
+ if (!key)
+ return nullptr;
+ for (const auto& spec : specs)
+ if (strcmp(key, spec.key) == 0)
+ return &spec;
+ return nullptr;
+}
+
+static sd_status_t check_key(sd_opt_t* params, const char* key, ParamType type, const ParamSpec** out) {
+ if (!params || !key)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "params and key are required");
+ *out = find_spec(key);
+ if (!*out)
+ return sd::api::error(SD_STATUS_UNKNOWN_KEY, key, "unknown parameter");
+ if (!((*out)->operations & params->operation))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, key, "parameter does not apply to this option object");
+ if ((*out)->type != type)
+ return sd::api::error(SD_STATUS_TYPE_MISMATCH, key, "incorrect setter type");
+ return SD_STATUS_OK;
+}
+
+static sd_status_t store_value(sd_opt_t* params, const ParamSpec& spec, ParamValue value) {
+ if (spec.type == ParamType::Int) {
+ const auto number = std::get<int64_t>(value);
+ if (number < spec.int_min || number > spec.int_max)
+ return sd::api::error(SD_STATUS_OUT_OF_RANGE, spec.key, "integer outside supported range");
+ }
+ if (spec.type == ParamType::Float) {
+ const auto number = std::get<double>(value);
+ if (!std::isfinite(number) || number < spec.float_min || number > spec.float_max)
+ return sd::api::error(SD_STATUS_OUT_OF_RANGE, spec.key, "float must be finite and within supported range");
+ }
+ if (spec.valid && !spec.valid(value))
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, spec.key, spec.invalid_reason);
+ params->values.insert_or_assign(spec.key, std::move(value));
+ return SD_STATUS_OK;
+}
+
+static sd_opt_t* create_opt(sd::api::Operation operation) {
+ sd_opt_t* params = nullptr;
+ sd::api::call([&] { params = new sd_opt_t{operation, {}}; return SD_STATUS_OK; });
+ return params;
+}
+
+sd_ctx_opt_t* sd_ctx_opt_create(void) {
+ return create_opt(sd::api::Context);
+}
+sd_img_gen_opt_t* sd_img_gen_opt_create(void) {
+ return create_opt(sd::api::Image);
+}
+sd_vid_gen_opt_t* sd_vid_gen_opt_create(void) {
+ return create_opt(sd::api::Video);
+}
+sd_upscaler_opt_t* sd_upscaler_opt_create(void) {
+ return create_opt(sd::api::Upscaler);
+}
+
+sd_upscale_opt_t* sd_upscale_opt_create(void) {
+ return create_opt(sd::api::Upscale);
+}
+
+sd_vid_upscaler_opt_t* sd_vid_upscaler_opt_create(void) {
+ return create_opt(sd::api::VidUpscaler);
+}
+
+sd_vid_upscale_opt_t* sd_vid_upscale_opt_create(void) {
+ return create_opt(sd::api::VidUpscale);
+}
+
+sd_adetailer_opt_t* sd_adetailer_opt_create(void) {
+ return create_opt(sd::api::ADetailer);
+}
+
+sd_adetail_opt_t* sd_adetail_opt_create(void) {
+ return create_opt(sd::api::ADetail);
+}
+
+void sd_opt_free(sd_opt_t* params) {
+ delete params;
+}
+
+sd_status_t sd_opt_set_bool(sd_opt_t* params, const char* key, bool value) {
+ return sd::api::call([&] {
+ const ParamSpec* spec = nullptr;
+ auto status = check_key(params, key, ParamType::Bool, &spec);
+ if (status != SD_STATUS_OK)
+ return status;
+ return store_value(params, *spec, ParamValue(value));
+ });
+}
+
+sd_status_t sd_opt_set_int(sd_opt_t* params, const char* key, int64_t value) {
+ return sd::api::call([&] {
+ const ParamSpec* spec = nullptr;
+ auto status = check_key(params, key, ParamType::Int, &spec);
+ if (status != SD_STATUS_OK)
+ return status;
+ return store_value(params, *spec, ParamValue(value));
+ });
+}
+
+sd_status_t sd_opt_set_float(sd_opt_t* params, const char* key, double value) {
+ return sd::api::call([&] {
+ const ParamSpec* spec = nullptr;
+ auto status = check_key(params, key, ParamType::Float, &spec);
+ if (status != SD_STATUS_OK)
+ return status;
+ return store_value(params, *spec, ParamValue(value));
+ });
+}
+
+sd_status_t sd_opt_set_string(sd_opt_t* params, const char* key, const char* value) {
+ return sd::api::call([&] {
+ const ParamSpec* spec = nullptr;
+ auto status = check_key(params, key, ParamType::String, &spec);
+ if (status != SD_STATUS_OK)
+ return status;
+ if (!value)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "string is NULL; use unset to restore defaults");
+ return store_value(params, *spec, std::string(value));
+ });
+}
+
+sd_status_t sd_opt_set_tensor(sd_opt_t* params, const char* key, const sd_tensor_t* tensor) {
+ return sd::api::call([&] {
+ const ParamSpec* spec = nullptr;
+ auto status = check_key(params, key, ParamType::Tensor, &spec);
+ if (status != SD_STATUS_OK)
+ return status;
+ if (!tensor)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "tensor is required");
+ TensorValue value;
+ value.tensor = sd::api::TensorRef(tensor);
+ try {
+ if (spec->video_tensor)
+ value.frames = sd::api::video_frames(*tensor);
+ else
+ sd::api::image_view(*tensor);
+ } catch (const std::invalid_argument& error) {
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, error.what());
+ }
+ return store_value(params, *spec, std::move(value));
+ });
+}
+
+template <typename T, typename Function>
+static sd_status_t set_array(sd_opt_t* params, const char* key, const T* values, size_t count, ParamType type, Function build) {
+ return sd::api::call([&] {
+ const ParamSpec* spec = nullptr;
+ auto status = check_key(params, key, type, &spec);
+ if (status != SD_STATUS_OK)
+ return status;
+ if (count > INT32_MAX)
+ return sd::api::error(SD_STATUS_OUT_OF_RANGE, key, "too many array elements");
+ if (!values && count)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "array is NULL with nonzero count");
+ return build(*spec);
+ });
+}
+
+sd_status_t sd_opt_set_tensors(sd_opt_t* params, const char* key, const sd_tensor_t* const* values, size_t count) {
+ return set_array(params, key, values, count, ParamType::Tensors, [&](const ParamSpec& spec) {
+ ImageValues copy;
+ copy.tensors.reserve(count);
+ copy.items.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ if (!values[i])
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "tensor is required");
+ copy.items.push_back(sd::api::image_view(*values[i]));
+ copy.tensors.emplace_back(values[i]);
+ }
+ return store_value(params, spec, std::move(copy));
+ });
+}
+
+sd_status_t sd_opt_set_floats(sd_opt_t* params, const char* key, const float* values, size_t count) {
+ return set_array(params, key, values, count, ParamType::Floats, [&](const ParamSpec& spec) {
+ std::vector<float> copy;
+ for (size_t i = 0; i < count; ++i) {
+ if (!std::isfinite(values[i]))
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "array values must be finite");
+ copy.push_back(values[i]);
+ }
+ return store_value(params, spec, std::move(copy));
+ });
+}
+
+sd_status_t sd_opt_set_ints(sd_opt_t* params, const char* key, const int32_t* values, size_t count) {
+ return set_array(params, key, values, count, ParamType::Ints, [&](const ParamSpec& spec) {
+ std::vector<int> copy;
+ for (size_t i = 0; i < count; ++i) {
+ copy.push_back(values[i]);
+ }
+ return store_value(params, spec, std::move(copy));
+ });
+}
+
+sd_status_t sd_opt_set_loras(sd_opt_t* params, const char* key, const sd_lora_t* values, size_t count) {
+ return set_array(params, key, values, count, ParamType::Loras, [&](const ParamSpec& spec) {
+ LoraValues copy;
+ copy.paths.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ if (!values[i].path || !*values[i].path || !std::isfinite(values[i].multiplier))
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "LoRA requires a path and finite multiplier");
+ copy.paths.emplace_back(values[i].path);
+ copy.items.push_back({values[i].is_high_noise, values[i].multiplier, copy.paths.back().c_str()});
+ }
+ return store_value(params, spec, std::move(copy));
+ });
+}
+
+sd_status_t sd_opt_set_embeddings(sd_opt_t* params, const char* key, const sd_embedding_t* values, size_t count) {
+ return set_array(params, key, values, count, ParamType::Embeddings, [&](const ParamSpec& spec) {
+ EmbeddingValues copy;
+ copy.strings.reserve(count * 2);
+ for (size_t i = 0; i < count; ++i) {
+ if (!values[i].name || !*values[i].name || !values[i].path || !*values[i].path)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "embedding requires a name and path");
+ copy.strings.emplace_back(values[i].name);
+ const char* name = copy.strings.back().c_str();
+ copy.strings.emplace_back(values[i].path);
+ copy.items.push_back({name, copy.strings.back().c_str()});
+ }
+ return store_value(params, spec, std::move(copy));
+ });
+}
+
+sd_status_t sd_opt_set_audios(sd_opt_t* params, const char* key, const sd_audio_t* values, size_t count) {
+ return set_array(params, key, values, count, ParamType::Audios, [&](const ParamSpec& spec) {
+ AudioValues copy;
+ copy.tensors.reserve(count);
+ copy.items.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ copy.items.push_back(sd::api::audio_view(values[i]));
+ copy.tensors.emplace_back(values[i].data);
+ }
+ return store_value(params, spec, std::move(copy));
+ });
+}
+
+sd_status_t sd_opt_set_videos(sd_opt_t* params, const char* key, const sd_video_t* values, size_t count) {
+ return set_array(params, key, values, count, ParamType::Videos, [&](const ParamSpec& spec) {
+ VideoValues copy;
+ copy.frames.reserve(count);
+ copy.items.reserve(count);
+ for (size_t i = 0; i < count; ++i) {
+ const auto& video = values[i];
+ if (!video.data || video.fps <= 0)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "video data and positive fps are required");
+ copy.frames.push_back(sd::api::video_frames(*video.data));
+ copy.tensors.emplace_back(video.data);
+ sd::Audio audio{};
+ if (video.audio.data) {
+ audio = sd::api::audio_view(video.audio);
+ copy.tensors.emplace_back(video.audio.data);
+ }
+ copy.items.push_back({copy.frames.back().data(), static_cast<int>(copy.frames.back().size()), video.fps, audio});
+ }
+ return store_value(params, spec, std::move(copy));
+ });
+}
+
+sd_status_t sd_opt_unset(sd_opt_t* params, const char* key) {
+ return sd::api::call([&] {
+ if (!params || !key)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, key, "params and key are required");
+ const auto* spec = find_spec(key);
+ if (!spec)
+ return sd::api::error(SD_STATUS_UNKNOWN_KEY, key, "unknown parameter");
+ if (!(spec->operations & params->operation))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, key, "parameter does not apply to this option object");
+ params->values.erase(key);
+ return SD_STATUS_OK;
+ });
+}
+
+bool sd::api::is_set(const sd_opt_t* params, const char* key) {
+ return params && params->values.count(key) != 0;
+}
+
+bool sd::api::has_prefix(const sd_opt_t* params, const char* prefix) {
+ if (!params)
+ return false;
+ const auto it = params->values.lower_bound(prefix);
+ return it != params->values.end() && it->first.compare(0, strlen(prefix), prefix) == 0;
+}
+
+sd_status_t sd::api::resolve(const sd_opt_t* params, Operation operation, OptConfig& config) {
+ if (!params)
+ return error(SD_STATUS_INVALID_ARGUMENT, nullptr, "params is required");
+ if (params->operation != operation)
+ return error(SD_STATUS_UNSUPPORTED, nullptr, "option object has the wrong purpose for this operation");
+ config = OptConfig{};
+ if (operation == ADetail)
+ config.image.strength = 0.4f;
+ for (const auto& entry : params->values) {
+ const auto* spec = find_spec(entry.first.c_str());
+ if (!(spec->operations & operation))
+ return error(SD_STATUS_UNSUPPORTED, spec->key, "parameter does not apply to this operation");
+ spec->apply(entry.second, config);
+ }
+ const auto& image = config.image;
+ const auto& video = config.video;
+ const bool sampling = operation == Image || operation == Video || operation == ADetail;
+ const auto& cache = operation == Video ? video.cache : image.cache;
+ if (sampling && cache.mode != SD_CACHE_DISABLED) {
+ if (cache.start_percent >= cache.end_percent)
+ return error(SD_STATUS_INVALID_ARGUMENT, "cache", "start_percent must precede end_percent");
+ }
+ if (sampling) {
+ const auto& sample = operation == Video ? video.sample_params : image.sample_params;
+ if (sample.guidance.slg.layer_start > sample.guidance.slg.layer_end)
+ return error(SD_STATUS_INVALID_ARGUMENT, "sample.guidance.slg", "layer_start must not exceed layer_end");
+ const auto& hires = operation == Video ? video.hires : image.hires;
+ if (hires.enabled && hires.upscaler != SD_HIRES_UPSCALER_NONE &&
+ hires.scale <= 0.f && hires.target_width <= 0 && hires.target_height <= 0)
+ return error(SD_STATUS_INVALID_ARGUMENT, "hires", "enabled hires requires a positive scale or target size");
+ }
+ if (operation == Video && video.high_noise_sample_params.guidance.slg.layer_start > video.high_noise_sample_params.guidance.slg.layer_end)
+ return error(SD_STATUS_INVALID_ARGUMENT, "high_noise_sample.guidance.slg", "layer_start must not exceed layer_end");
+ if ((operation == Upscaler || operation == ADetailer) && (!config.context.model_path || !*config.context.model_path))
+ return error(SD_STATUS_INVALID_ARGUMENT, "model_path", "model path is required");
+ if (operation == VidUpscaler) {
+ const auto& load = config.post.video_upscaler;
+ if (!load.diffusion_model_path || !*load.diffusion_model_path || !load.vae_path || !*load.vae_path || load.upscale_factor <= 0)
+ return error(SD_STATUS_INVALID_ARGUMENT, nullptr, "diffusion_model_path, vae_path and upscale_factor are required");
+ }
+ if ((operation == Upscale || operation == ADetail) && !config.post.image.data)
+ return error(SD_STATUS_INVALID_ARGUMENT, "image", "input image is required");
+ if (operation == Upscale && config.post.image.channel != 3)
+ return error(SD_STATUS_INVALID_ARGUMENT, "image", "upscaling requires an RGB image");
+ if (operation == VidUpscale) {
+ const auto& input = config.post.video_upscale;
+ if (!input.frames || input.frame_count <= 0)
+ return error(SD_STATUS_INVALID_ARGUMENT, "frames", "video upscaling requires nonempty frames");
+ for (int i = 0; i < input.frame_count; ++i)
+ if (input.frames[i].channel != 3 || input.frames[i].width != input.frames[0].width || input.frames[i].height != input.frames[0].height)
+ return error(SD_STATUS_INVALID_ARGUMENT, "frames", "frames must be RGB and have equal dimensions");
+ }
+ return SD_STATUS_OK;
+}
diff --git a/src/api/options.h b/src/api/options.h
new file mode 100644
index 0000000..6517829
--- /dev/null
+++ b/src/api/options.h
@@ -0,0 +1,25 @@
+#ifndef SD_OPTIONS_H
+#define SD_OPTIONS_H
+
+#include "api.h"
+#include "config.h"
+
+namespace sd::api {
+ enum Operation {
+ Context = 1,
+ Image = 2,
+ Video = 4,
+ Upscaler = 8,
+ Upscale = 16,
+ VidUpscaler = 32,
+ VidUpscale = 64,
+ ADetailer = 128,
+ ADetail = 256
+ };
+
+ sd_status_t resolve(const sd_opt_t* params, Operation operation, OptConfig& config);
+ bool is_set(const sd_opt_t* params, const char* key);
+ bool has_prefix(const sd_opt_t* params, const char* prefix);
+}
+
+#endif // SD_OPTIONS_H
diff --git a/src/api/result.cpp b/src/api/result.cpp
new file mode 100644
index 0000000..388f948
--- /dev/null
+++ b/src/api/result.cpp
@@ -0,0 +1,243 @@
+#include "result.h"
+#include "api.h"
+
+#include <cstdlib>
+#include <cstring>
+#include <utility>
+
+namespace sd {
+ using nlohmann::json;
+
+ char* copy_generation_info(const json& info) {
+ const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace);
+ char* result = static_cast<char*>(malloc(text.size() + 1));
+ if (result)
+ memcpy(result, text.c_str(), text.size() + 1);
+ return result;
+ }
+
+ bool make_image_generation_result(sd::Image* images, int count, const std::vector<json>& infos, sd::ImageGenerationResult** result_out) {
+ if (!result_out || !images || count <= 0 || infos.size() != static_cast<size_t>(count)) {
+ sd::free_images(images, count);
+ return false;
+ }
+ auto* result = static_cast<sd::ImageGenerationResult*>(calloc(1, sizeof(sd::ImageGenerationResult)));
+ if (!result) {
+ sd::free_images(images, count);
+ return false;
+ }
+ result->items = static_cast<sd::ImageResult*>(calloc(count, sizeof(sd::ImageResult)));
+ if (!result->items) {
+ free(result);
+ sd::free_images(images, count);
+ return false;
+ }
+ result->count = count;
+ try {
+ for (int i = 0; i < count; ++i) {
+ result->items[i].image = images[i];
+ images[i] = {};
+ result->items[i].generation_info = copy_generation_info(infos[i]);
+ if (!result->items[i].generation_info) {
+ sd::free_images(images, count);
+ sd::free_image_generation_result(result);
+ return false;
+ }
+ }
+ } catch (...) {
+ sd::free_images(images, count);
+ sd::free_image_generation_result(result);
+ return false;
+ }
+ free(images);
+ *result_out = result;
+ return true;
+ }
+}
+
+void sd::free_image_generation_result(sd::ImageGenerationResult* result) {
+ if (!result)
+ return;
+ for (int i = 0; i < result->count; ++i) {
+ free(result->items[i].image.data);
+ free(result->items[i].generation_info);
+ }
+ free(result->items);
+ free(result);
+}
+
+void sd::free_video_generation_result(sd::VideoGenerationResult* result) {
+ if (!result)
+ return;
+ sd::free_images(result->frames, result->frame_count);
+ sd::free_audio(result->audio);
+ free(result->generation_info);
+ free(result);
+}
+
+static sd::api::TensorRef consume_image(sd::Image& image) {
+ std::vector<int64_t> shape{image.channel, image.width, image.height};
+ auto* data = image.data;
+ image.data = nullptr;
+ return sd::api::own_tensor(SD_TENSOR_U8, std::move(shape), data);
+}
+
+bool sd::make_image_result(ImageGenerationResult* images, sd_result_t** result_out) {
+ if (result_out)
+ *result_out = nullptr;
+ std::unique_ptr<ImageGenerationResult, decltype(&free_image_generation_result)> owner(images, free_image_generation_result);
+ if (!result_out || !images)
+ return false;
+ auto result = std::make_unique<sd_result_t>();
+ result->kind = SD_RESULT_IMAGE;
+ result->images.reserve(images->count);
+ result->generation_infos.reserve(images->count);
+ for (int i = 0; i < images->count; ++i) {
+ result->images.push_back(consume_image(images->items[i].image));
+ result->generation_infos.emplace_back(images->items[i].generation_info ? images->items[i].generation_info : "");
+ }
+ *result_out = result.release();
+ return true;
+}
+
+bool sd::make_image_result(sd::Image* images, int count, const std::vector<nlohmann::json>& infos, sd_result_t** result_out) {
+ if (result_out)
+ *result_out = nullptr;
+ ImageGenerationResult* raw = nullptr;
+ if (!make_image_generation_result(images, count, infos, &raw))
+ return false;
+ return make_image_result(raw, result_out);
+}
+
+bool sd::make_video_result(VideoGenerationResult* video, sd_result_t** result_out) {
+ if (result_out)
+ *result_out = nullptr;
+ std::unique_ptr<VideoGenerationResult, decltype(&free_video_generation_result)> owner(video, free_video_generation_result);
+ if (!result_out || !video)
+ return false;
+ auto result = std::make_unique<sd_result_t>();
+ result->kind = SD_RESULT_VIDEO;
+ result->fps = video->fps;
+ result->images.reserve(video->frame_count);
+ for (int i = 0; i < video->frame_count; ++i)
+ result->images.push_back(consume_image(video->frames[i]));
+ result->generation_infos.emplace_back(video->generation_info ? video->generation_info : "");
+ if (video->audio) {
+ auto& audio = *video->audio;
+ std::vector<int64_t> shape{audio.channels, static_cast<int64_t>(audio.sample_count)};
+ auto* data = audio.data;
+ audio.data = nullptr;
+ result->audio_tensor = sd::api::own_tensor(SD_TENSOR_F32, std::move(shape), data);
+ result->audio = {audio.sample_rate, result->audio_tensor.get()};
+ }
+ *result_out = result.release();
+ return true;
+}
+
+void sd::free_images(sd::Image* result_images, int num_images) {
+ if (result_images == nullptr) {
+ return;
+ }
+
+ for (int i = 0; i < num_images; ++i) {
+ if (result_images[i].data != nullptr) {
+ free(result_images[i].data);
+ result_images[i].data = nullptr;
+ }
+ }
+
+ free(result_images);
+}
+
+void sd::free_audio(sd::Audio* audio) {
+ if (audio == nullptr) {
+ return;
+ }
+ free(audio->data);
+ audio->data = nullptr;
+ free(audio);
+}
+
+void sd::free_image(sd::Image* image) {
+ if (image) {
+ free(image->data);
+ *image = {};
+ }
+}
+
+void sd_result_free(sd_result_t* result) {
+ delete result;
+}
+
+sd_result_kind_t sd_result_get_kind(const sd_result_t* result) {
+ sd::api::clear_error();
+ if (!result) {
+ sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result is required");
+ return SD_RESULT_INVALID;
+ }
+ return result->kind;
+}
+
+size_t sd_result_get_image_count(const sd_result_t* result) {
+ if (!sd_result_get_kind(result))
+ return 0;
+ return result->images.size();
+}
+
+const sd_tensor_t* sd_result_get_image(const sd_result_t* result, size_t index) {
+ if (index >= sd_result_get_image_count(result)) {
+ sd::api::error(result ? SD_STATUS_OUT_OF_RANGE : SD_STATUS_INVALID_ARGUMENT, nullptr, "invalid result or image index");
+ return nullptr;
+ }
+ return result->images[index].get();
+}
+
+const char* sd_result_get_generation_info(const sd_result_t* result, size_t index) {
+ if (!sd_result_get_image(result, index))
+ return nullptr;
+ return result->generation_infos[result->kind == SD_RESULT_VIDEO ? 0 : index].c_str();
+}
+
+const sd_audio_t* sd_result_get_audio(const sd_result_t* result) {
+ if (!sd_result_get_kind(result))
+ return nullptr;
+ return result->audio.data ? &result->audio : nullptr;
+}
+
+int sd_result_get_fps(const sd_result_t* result) {
+ if (!sd_result_get_kind(result))
+ return 0;
+ return result->fps;
+}
+
+const sd_video_t* sd_result_get_video(const sd_result_t* result) {
+ const sd_video_t* output = nullptr;
+ sd::api::call([&] {
+ if (!result)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "result is required");
+ if (result->kind != SD_RESULT_VIDEO)
+ return sd::api::error(SD_STATUS_TYPE_MISMATCH, nullptr, "result is not a video");
+ std::lock_guard<std::mutex> lock(result->video_mutex);
+ if (!result->video_tensor) {
+ if (result->images.empty())
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "video has no frames");
+ const auto& first = *result->images.front().get();
+ auto shape = first.shape;
+ shape.push_back(static_cast<int64_t>(result->images.size()));
+ for (const auto& frame : result->images)
+ if (frame->type != first.type || frame->shape != first.shape)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "video frames must have identical shapes and types");
+ auto packed = sd::api::allocate_tensor(first.type, std::move(shape));
+ auto* destination = static_cast<uint8_t*>(packed.get()->mutable_data());
+ for (const auto& frame : result->images) {
+ memcpy(destination, frame->data(), frame->bytes);
+ destination += frame->bytes;
+ }
+ result->video_tensor = std::move(packed);
+ result->video = {result->video_tensor.get(), result->fps, result->audio};
+ }
+ output = &result->video;
+ return SD_STATUS_OK;
+ });
+ return output;
+}
diff --git a/src/api/result.h b/src/api/result.h
new file mode 100644
index 0000000..452055b
--- /dev/null
+++ b/src/api/result.h
@@ -0,0 +1,57 @@
+#ifndef SD_RESULT_H
+#define SD_RESULT_H
+
+#include <memory>
+#include <mutex>
+#include <string>
+#include <vector>
+#include "tensor.h"
+
+#include "json.hpp"
+#include "stable-diffusion.h"
+
+namespace sd {
+ struct ImageResult {
+ sd::Image image;
+ // Owned UTF-8 JSON using the sdcpp.generation/v1 schema.
+ char* generation_info;
+ };
+
+ struct ImageGenerationResult {
+ sd::ImageResult* items;
+ int count;
+ };
+
+ struct VideoGenerationResult {
+ sd::Image* frames;
+ int frame_count;
+ sd::Audio* audio;
+ int fps;
+ char* generation_info;
+ };
+
+ void free_image_generation_result(ImageGenerationResult* result);
+ void free_video_generation_result(VideoGenerationResult* result);
+ void free_images(sd::Image* images, int count);
+
+ char* copy_generation_info(const nlohmann::json& info);
+ // Consumes images even on failure.
+ bool make_image_generation_result(sd::Image* images, int count, const std::vector<nlohmann::json>& infos, sd::ImageGenerationResult** result_out);
+ bool make_image_result(ImageGenerationResult* images, sd_result_t** result_out);
+ bool make_image_result(sd::Image* images, int count, const std::vector<nlohmann::json>& infos, sd_result_t** result_out);
+ bool make_video_result(VideoGenerationResult* video, sd_result_t** result_out);
+}
+
+struct sd_result {
+ sd_result_kind_t kind = SD_RESULT_INVALID;
+ std::vector<sd::api::TensorRef> images;
+ std::vector<std::string> generation_infos;
+ sd::api::TensorRef audio_tensor;
+ sd_audio_t audio{};
+ int fps = 0;
+ mutable std::mutex video_mutex;
+ mutable sd::api::TensorRef video_tensor;
+ mutable sd_video_t video{};
+};
+
+#endif // SD_RESULT_H
diff --git a/src/api/tensor.cpp b/src/api/tensor.cpp
new file mode 100644
index 0000000..7567535
--- /dev/null
+++ b/src/api/tensor.cpp
@@ -0,0 +1,191 @@
+#include "tensor.h"
+#include "api.h"
+
+#include <climits>
+#include <cstdlib>
+#include <cstring>
+#include <limits>
+#include <stdexcept>
+
+static size_t element_size(sd_tensor_type_t type) {
+ switch (type) {
+ case SD_TENSOR_U8:
+ return sizeof(uint8_t);
+ case SD_TENSOR_I32:
+ return sizeof(int32_t);
+ case SD_TENSOR_I64:
+ return sizeof(int64_t);
+ case SD_TENSOR_F32:
+ return sizeof(float);
+ case SD_TENSOR_F64:
+ return sizeof(double);
+ default:
+ throw std::invalid_argument("invalid tensor type");
+ }
+}
+
+size_t sd::api::tensor_byte_size(sd_tensor_type_t type, const std::vector<int64_t>& shape) {
+ if (shape.size() > 8)
+ throw std::invalid_argument("tensor rank must not exceed 8");
+ size_t count = 1;
+ for (int64_t dim : shape) {
+ if (dim < 0 || (count && uint64_t(dim) > SIZE_MAX / count))
+ throw std::invalid_argument("negative or overflowing tensor dimensions");
+ count *= static_cast<size_t>(dim);
+ }
+ const size_t size = element_size(type);
+ if (count > uint64_t(INT64_MAX) || count > SIZE_MAX / size)
+ throw std::invalid_argument("tensor byte size overflow");
+ return count * size;
+}
+
+static sd_tensor_t::Storage allocate_storage(sd_tensor_type_t type, std::vector<int64_t> shape, size_t bytes) {
+ if (bytes && shape.empty())
+ shape.push_back(1);
+ if (!bytes)
+ shape.clear();
+ switch (type) {
+ case SD_TENSOR_U8:
+ return sd::Tensor<uint8_t>(std::move(shape));
+ case SD_TENSOR_I32:
+ return sd::Tensor<int32_t>(std::move(shape));
+ case SD_TENSOR_I64:
+ return sd::Tensor<int64_t>(std::move(shape));
+ case SD_TENSOR_F32:
+ return sd::Tensor<float>(std::move(shape));
+ case SD_TENSOR_F64:
+ return sd::Tensor<double>(std::move(shape));
+ default:
+ throw std::invalid_argument("invalid tensor type");
+ }
+}
+
+sd_tensor_t::sd_tensor_t(sd_tensor_type_t type, std::vector<int64_t> shape)
+ : type(type), shape(std::move(shape)), bytes(sd::api::tensor_byte_size(type, this->shape)), storage_(allocate_storage(type, this->shape, bytes)) {}
+
+static std::vector<int64_t> validated_shape(sd_tensor_type_t type, std::vector<int64_t> shape, const void* data, size_t size) {
+ if (size != sd::api::tensor_byte_size(type, shape) || (size && !data))
+ throw std::invalid_argument("tensor data size does not match type and shape");
+ return shape;
+}
+
+sd_tensor_t::sd_tensor_t(sd_tensor_type_t type, std::vector<int64_t> shape, const void* data, size_t size)
+ : sd_tensor_t(type, validated_shape(type, std::move(shape), data, size)) {
+ if (bytes)
+ memcpy(mutable_data(), data, bytes);
+}
+
+sd::api::TensorRef sd::api::allocate_tensor(sd_tensor_type_t type, std::vector<int64_t> shape) {
+ return TensorRef::adopt(new sd_tensor_t(type, std::move(shape)));
+}
+
+sd::api::TensorRef sd::api::own_tensor(sd_tensor_type_t type, std::vector<int64_t> shape, void* data) {
+ std::unique_ptr<void, decltype(&std::free)> owner(data, std::free);
+ const size_t bytes = tensor_byte_size(type, shape);
+ return TensorRef::adopt(new sd_tensor_t(type, std::move(shape), data, bytes));
+}
+
+sd::Image sd::api::image_view(const sd_tensor_t& tensor) {
+ const auto& s = tensor.shape;
+ if (tensor.type != SD_TENSOR_U8 || s.size() != 3 || s[0] < 1 || s[0] > 4 ||
+ s[1] <= 0 || s[2] <= 0 || s[1] > INT32_MAX || s[2] > INT32_MAX || !tensor.data())
+ throw std::invalid_argument("image requires U8 [channels,width,height], channels in [1,4], and positive dimensions");
+ return {uint32_t(s[1]), uint32_t(s[2]), uint32_t(s[0]), const_cast<uint8_t*>(static_cast<const uint8_t*>(tensor.data()))};
+}
+
+sd::Audio sd::api::audio_view(const sd_audio_t& audio) {
+ if (!audio.data || audio.sample_rate == 0)
+ throw std::invalid_argument("audio requires samples and a positive sample rate");
+ const auto& t = *audio.data;
+ if (t.type != SD_TENSOR_F32 || t.shape.size() != 2 || t.shape[0] <= 0 || t.shape[0] > UINT32_MAX || t.shape[1] <= 0 || !t.data())
+ throw std::invalid_argument("audio requires F32 [channels,samples] with positive dimensions");
+ return {audio.sample_rate, uint32_t(t.shape[0]), uint64_t(t.shape[1]), const_cast<float*>(static_cast<const float*>(t.data()))};
+}
+
+std::vector<sd::Image> sd::api::video_frames(const sd_tensor_t& tensor) {
+ const auto& s = tensor.shape;
+ if (tensor.type != SD_TENSOR_U8 || s.size() != 4 || s[0] < 1 || s[0] > 4 || s[1] <= 0 || s[2] <= 0 || s[3] <= 0 ||
+ s[1] > INT32_MAX || s[2] > INT32_MAX || s[3] > INT32_MAX || !tensor.data())
+ throw std::invalid_argument("video requires U8 [channels,width,height,frames] with positive dimensions");
+ const size_t stride = size_t(s[0]) * size_t(s[1]) * size_t(s[2]);
+ auto* data = const_cast<uint8_t*>(static_cast<const uint8_t*>(tensor.data()));
+ std::vector<sd::Image> result;
+ result.reserve(size_t(s[3]));
+ for (int64_t i = 0; i < s[3]; ++i)
+ result.push_back({uint32_t(s[1]), uint32_t(s[2]), uint32_t(s[0]), data + size_t(i) * stride});
+ return result;
+}
+
+void sd::api::preview_images(const std::function<void(int, int, const sd_tensor_t* const*, bool, void*)>& callback, int step, int count, const sd::Image* images, bool noisy, void* data) {
+ std::vector<TensorRef> tensors;
+ std::vector<const sd_tensor_t*> views;
+ tensors.reserve(count);
+ views.reserve(count);
+ for (int i = 0; i < count; ++i) {
+ const auto& image = images[i];
+ const int64_t shape[] = {image.channel, image.width, image.height};
+ auto* tensor = sd_tensor_create(SD_TENSOR_U8, 3, shape, image.data, size_t(image.channel) * image.width * image.height);
+ if (!tensor)
+ throw std::bad_alloc();
+ tensors.push_back(TensorRef::adopt(tensor));
+ views.push_back(tensors.back().get());
+ }
+ callback(step, count, views.data(), noisy, data);
+}
+
+sd_tensor_t* sd_tensor_create(sd_tensor_type_t type, size_t rank, const int64_t* shape, const void* data, size_t data_size) {
+ sd_tensor_t* result = nullptr;
+ sd::api::call([&] {
+ if (rank > 8 || (rank && !shape))
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "invalid tensor shape");
+ std::vector<int64_t> dimensions;
+ if (rank)
+ dimensions.assign(shape, shape + rank);
+ const size_t bytes = sd::api::tensor_byte_size(type, dimensions);
+ if (data_size != bytes || (bytes && !data))
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "tensor data size does not match type and shape");
+ result = new sd_tensor_t(type, std::move(dimensions), data, data_size);
+ return SD_STATUS_OK;
+ });
+ return result;
+}
+
+sd_tensor_t* sd_tensor_retain(const sd_tensor_t* tensor) {
+ if (tensor)
+ tensor->references.fetch_add(1, std::memory_order_relaxed);
+ return const_cast<sd_tensor_t*>(tensor);
+}
+
+void sd_tensor_free(sd_tensor_t* tensor) {
+ if (tensor && tensor->references.fetch_sub(1, std::memory_order_acq_rel) == 1)
+ delete tensor;
+}
+
+static bool valid_tensor(const sd_tensor_t* tensor) {
+ sd::api::clear_error();
+ if (!tensor)
+ sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "tensor is required");
+ return tensor != nullptr;
+}
+
+sd_tensor_type_t sd_tensor_type(const sd_tensor_t* tensor) {
+ return valid_tensor(tensor) ? tensor->type : 0;
+}
+size_t sd_tensor_rank(const sd_tensor_t* tensor) {
+ return valid_tensor(tensor) ? tensor->shape.size() : 0;
+}
+size_t sd_tensor_byte_size(const sd_tensor_t* tensor) {
+ return valid_tensor(tensor) ? tensor->bytes : 0;
+}
+const void* sd_tensor_data(const sd_tensor_t* tensor) {
+ return valid_tensor(tensor) ? tensor->data() : nullptr;
+}
+int64_t sd_tensor_dim(const sd_tensor_t* tensor, size_t axis) {
+ if (!valid_tensor(tensor))
+ return 0;
+ if (axis >= tensor->shape.size()) {
+ sd::api::error(SD_STATUS_OUT_OF_RANGE, nullptr, "invalid tensor axis");
+ return 0;
+ }
+ return tensor->shape[axis];
+}
diff --git a/src/api/tensor.h b/src/api/tensor.h
new file mode 100644
index 0000000..9d57287
--- /dev/null
+++ b/src/api/tensor.h
@@ -0,0 +1,93 @@
+#ifndef __SD_API_TENSOR_H__
+#define __SD_API_TENSOR_H__
+
+#include <atomic>
+#include <cstring>
+#include <functional>
+#include <memory>
+#include <stdexcept>
+#include <type_traits>
+#include <variant>
+#include <vector>
+#include "core/tensor.hpp"
+
+#include "core/media.h"
+#include "stable-diffusion.h"
+
+namespace sd::api {
+ size_t tensor_byte_size(sd_tensor_type_t type, const std::vector<int64_t>& shape);
+}
+
+struct sd_tensor_t {
+ using Storage = std::variant<sd::Tensor<uint8_t>, sd::Tensor<int32_t>, sd::Tensor<int64_t>, sd::Tensor<float>, sd::Tensor<double>>;
+ mutable std::atomic<size_t> references{1};
+ sd_tensor_type_t type;
+ std::vector<int64_t> shape;
+ size_t bytes;
+
+private:
+ Storage storage_;
+
+public:
+ sd_tensor_t(sd_tensor_type_t type, std::vector<int64_t> shape);
+ sd_tensor_t(sd_tensor_type_t type, std::vector<int64_t> shape, const void* data, size_t size);
+
+ template <typename T>
+ explicit sd_tensor_t(sd::Tensor<T> tensor)
+ : type(std::is_same_v<T, uint8_t> ? SD_TENSOR_U8 : std::is_same_v<T, int32_t> ? SD_TENSOR_I32
+ : std::is_same_v<T, int64_t> ? SD_TENSOR_I64
+ : std::is_same_v<T, float> ? SD_TENSOR_F32
+ : SD_TENSOR_F64),
+ shape(tensor.shape()),
+ bytes(sd::api::tensor_byte_size(type, shape)),
+ storage_(std::move(tensor)) {
+ static_assert(std::is_same_v<T, uint8_t> || std::is_same_v<T, int32_t> || std::is_same_v<T, int64_t> || std::is_same_v<T, float> || std::is_same_v<T, double>, "unsupported tensor type");
+ if (std::get<sd::Tensor<T>>(storage_).numel() != static_cast<int64_t>(bytes / sizeof(T)))
+ throw std::invalid_argument("tensor shape and storage size do not match");
+ }
+
+ const void* data() const {
+ return bytes ? std::visit([](const auto& tensor) -> const void* { return tensor.data(); }, storage_) : nullptr;
+ }
+ void* mutable_data() {
+ return bytes ? std::visit([](auto& tensor) -> void* { return tensor.data(); }, storage_) : nullptr;
+ }
+};
+
+namespace sd::api {
+ class TensorRef {
+ sd_tensor_t* tensor_ = nullptr;
+
+ public:
+ TensorRef() = default;
+ explicit TensorRef(const sd_tensor_t* tensor)
+ : tensor_(sd_tensor_retain(tensor)) {}
+ static TensorRef adopt(sd_tensor_t* tensor) {
+ TensorRef result;
+ result.tensor_ = tensor;
+ return result;
+ }
+ TensorRef(const TensorRef& other)
+ : TensorRef(other.tensor_) {}
+ TensorRef(TensorRef&& other) noexcept
+ : tensor_(other.tensor_) { other.tensor_ = nullptr; }
+ TensorRef& operator=(TensorRef other) noexcept {
+ std::swap(tensor_, other.tensor_);
+ return *this;
+ }
+ ~TensorRef() { sd_tensor_free(tensor_); }
+ sd_tensor_t* get() const { return tensor_; }
+ const sd_tensor_t* operator->() const { return tensor_; }
+ explicit operator bool() const { return tensor_ != nullptr; }
+ };
+
+ size_t tensor_byte_size(sd_tensor_type_t type, const std::vector<int64_t>& shape);
+ TensorRef own_tensor(sd_tensor_type_t type, std::vector<int64_t> shape, void* data);
+ TensorRef allocate_tensor(sd_tensor_type_t type, std::vector<int64_t> shape);
+ sd::Image image_view(const sd_tensor_t& tensor);
+ sd::Audio audio_view(const sd_audio_t& audio);
+ std::vector<sd::Image> video_frames(const sd_tensor_t& tensor);
+ void preview_images(const std::function<void(int, int, const sd_tensor_t* const*, bool, void*)>& callback, int step, int count, const sd::Image* images, bool noisy, void* data);
+}
+
+#endif // __SD_API_TENSOR_H__
diff --git a/src/conditioning/conditioner.hpp b/src/conditioning/conditioner.hpp
index eeebb77..808375c 100644
--- a/src/conditioning/conditioner.hpp
+++ b/src/conditioning/conditioner.hpp
@@ -1997,13 +1997,13 @@ struct LLMEmbedder : public Conditioner {
: version(version) {
if (!tokenizers.has(TokenizerConfig::MAIN)) {
if (sd_version_is_lens(version)) {
- throw std::runtime_error("Lens requires an external GPT-OSS tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer");
+ throw std::runtime_error("Lens requires an external GPT-OSS tokenizer.json; pass --tokenizer FILE or set the tokenizer context option");
}
if (sd_version_is_pid(version)) {
- throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer");
+ throw std::runtime_error("PiD requires an external Gemma 2 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option");
}
if (version == VERSION_MOSS_IMAGE) {
- throw std::runtime_error("MOSS-Image requires the official Qwen3.5 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer");
+ throw std::runtime_error("MOSS-Image requires the official Qwen3.5 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option");
}
}
LLM::LLMArch arch = LLM::LLMArch::QWEN2_5_VL;
@@ -3643,7 +3643,7 @@ struct LLaDAImageEmbedder : public Conditioner {
text_projection_prefix(text_projection_prefix),
sigvq_prefix(sigvq_prefix) {
if (!tokenizers.has(TokenizerConfig::MAIN)) {
- throw std::runtime_error("LLaDA-Image requires an external LLaDA2 tokenizer.json; pass --tokenizer FILE or set sd_ctx_params_t::tokenizer");
+ throw std::runtime_error("LLaDA-Image requires an external LLaDA2 tokenizer.json; pass --tokenizer FILE or set the tokenizer context option");
}
llm = std::make_shared<LLM::LLMRunner>(LLM::LLMArch::LLADA2_MOE,
backend,
diff --git a/src/config.cpp b/src/config.cpp
new file mode 100644
index 0000000..5cffb85
--- /dev/null
+++ b/src/config.cpp
@@ -0,0 +1,133 @@
+#include "config.h"
+
+#include <cmath>
+
+namespace sd {
+
+ SampleConfig::SampleConfig() {
+ guidance.txt_cfg = 7.0f;
+ guidance.img_cfg = INFINITY;
+ guidance.distilled_guidance = 3.5f;
+ guidance.slg.layers = nullptr;
+ guidance.slg.layer_count = 0;
+ guidance.slg.layer_start = 0.01f;
+ guidance.slg.layer_end = 0.2f;
+ guidance.slg.scale = 0.f;
+ scheduler = SCHEDULER_COUNT;
+ sample_method = SAMPLE_METHOD_COUNT;
+ shifted_timestep = 0;
+ sample_steps = 20;
+ eta = INFINITY;
+ custom_sigmas = nullptr;
+ custom_sigmas_count = 0;
+ flow_shift = INFINITY;
+ extra_sample_args = nullptr;
+ }
+
+ CacheConfig::CacheConfig() {
+ mode = SD_CACHE_DISABLED;
+ reuse_threshold = INFINITY;
+ start_percent = 0.15f;
+ end_percent = 0.95f;
+ error_decay_rate = 1.0f;
+ use_relative_threshold = true;
+ reset_error_on_compute = true;
+ Fn_compute_blocks = 8;
+ Bn_compute_blocks = 0;
+ residual_diff_threshold = 0.08f;
+ max_warmup_steps = 8;
+ max_cached_steps = -1;
+ max_continuous_cached_steps = -1;
+ taylorseer_n_derivatives = 1;
+ taylorseer_skip_interval = 1;
+ scm_mask = nullptr;
+ scm_policy_dynamic = true;
+ spectrum_w = 0.40f;
+ spectrum_m = 3;
+ spectrum_lam = 1.0f;
+ spectrum_window_size = 2;
+ spectrum_flex_window = 0.50f;
+ spectrum_warmup_steps = 4;
+ spectrum_stop_percent = 0.9f;
+ }
+
+ HiresConfig::HiresConfig() {
+ enabled = false;
+ upscaler = SD_HIRES_UPSCALER_LATENT;
+ model_path = nullptr;
+ scale = 2.f;
+ target_width = 0;
+ target_height = 0;
+ steps = 0;
+ denoising_strength = 0.7f;
+ upscale_tile_size = 128;
+ custom_sigmas = nullptr;
+ custom_sigmas_count = 0;
+ }
+
+ ContextConfig::ContextConfig() {
+ n_threads = sd_get_num_physical_cores();
+ conditioning_cache_size = 4;
+ wtype = SD_TYPE_COUNT;
+ rng_type = CUDA_RNG;
+ sampler_rng_type = RNG_TYPE_COUNT;
+ prediction = PREDICTION_COUNT;
+ lora_apply_mode = LORA_APPLY_AUTO;
+ max_vram = nullptr;
+ disable_prefetch = false;
+ disable_segmented_compute = false;
+ batched_cfg = true;
+ eager_load = false;
+ enable_mmap = false;
+ flash_attn_mode = nullptr;
+ conv_direct_mode = nullptr;
+ diffusion_flash_attn = false;
+ sage_attn = false;
+ linear_scale = 0.f;
+ attn_scale = 0.f;
+ vae_format = SD_VAE_FORMAT_AUTO;
+ backend = nullptr;
+ params_backend = nullptr;
+ split_mode = nullptr;
+ auto_fit = true;
+ rpc_servers = nullptr;
+ model_args = nullptr;
+ pulid_weights_path = nullptr;
+ }
+
+ ImageConfig::ImageConfig() {
+ clip_skip = -1;
+ ref_images_count = 0;
+ ref_image_args = "";
+ width = 512;
+ height = 512;
+ strength = 0.75f;
+ seed = -1;
+ batch_count = 1;
+ control_strength = 0.9f;
+ ip_adapter_strength = 1.0f;
+ qwen_image_layers = 3;
+ circular_x = false;
+ circular_y = false;
+ pm_params = {nullptr, 0, nullptr, 20.f};
+ pulid_params = {nullptr, 1.0f};
+ vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
+ }
+
+ VideoConfig::VideoConfig() {
+ high_noise_sample_params.sample_steps = -1;
+ width = 512;
+ height = 512;
+ strength = 0.75f;
+ seed = -1;
+ video_frames = 6;
+ fps = 16;
+ moe_boundary = 0.875f;
+ vace_strength = 1.f;
+ vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
+ circular_x = false;
+ circular_y = false;
+ ref_image_args = "";
+ }
+
+} // namespace sd
diff --git a/src/config.h b/src/config.h
new file mode 100644
index 0000000..7331ccc
--- /dev/null
+++ b/src/config.h
@@ -0,0 +1,295 @@
+#ifndef SD_CONFIG_H
+#define SD_CONFIG_H
+
+#include "core/media.h"
+
+#include "core/option_types.h"
+#include "stable-diffusion.h"
+
+namespace sd {
+ struct TilingConfig {
+ bool enabled;
+ bool temporal_tiling;
+ // Spatial tile dimensions in image pixels for both encode and decode; 0 uses 256.
+ int tile_size_w;
+ int tile_size_h;
+ float target_overlap;
+ // Positive values override tile_size: <= 1 is a dimension fraction, > 1 a target tile count.
+ float rel_size_w;
+ float rel_size_h;
+ const char* extra_tiling_args;
+ };
+
+ struct ImagePreprocessConfig {
+ // Semicolon-separated target=...,key=value rules. NULL preserves defaults.
+ const char* rules;
+ };
+
+ struct SLGConfig {
+ int* layers;
+ size_t layer_count;
+ float layer_start;
+ float layer_end;
+ float scale;
+ };
+
+ struct GuidanceConfig {
+ float txt_cfg;
+ float img_cfg;
+ float distilled_guidance;
+ sd::SLGConfig slg;
+ };
+
+ struct SampleConfig {
+ sd::GuidanceConfig guidance;
+ enum scheduler_t scheduler;
+ enum sample_method_t sample_method;
+ int sample_steps;
+ float eta;
+ int shifted_timestep;
+ float* custom_sigmas;
+ int custom_sigmas_count;
+ float flow_shift;
+ const char* extra_sample_args;
+
+ SampleConfig();
+ };
+
+ struct PhotoMakerConfig {
+ sd::Image* id_images;
+ int id_images_count;
+ const char* id_embed_path;
+ float style_strength;
+ };
+
+ struct PuLIDConfig {
+ const char* id_embedding_path;
+ float id_weight;
+ };
+
+ struct CacheConfig {
+ enum sd_cache_mode_t mode;
+ float reuse_threshold;
+ float start_percent;
+ float end_percent;
+ float error_decay_rate;
+ bool use_relative_threshold;
+ bool reset_error_on_compute;
+ int Fn_compute_blocks;
+ int Bn_compute_blocks;
+ float residual_diff_threshold;
+ int max_warmup_steps;
+ int max_cached_steps;
+ int max_continuous_cached_steps;
+ int taylorseer_n_derivatives;
+ int taylorseer_skip_interval;
+ const char* scm_mask;
+ bool scm_policy_dynamic;
+ float spectrum_w;
+ int spectrum_m;
+ float spectrum_lam;
+ int spectrum_window_size;
+ float spectrum_flex_window;
+ int spectrum_warmup_steps;
+ float spectrum_stop_percent;
+
+ CacheConfig();
+ };
+
+ struct HiresConfig {
+ bool enabled;
+ enum sd_hires_upscaler_t upscaler;
+ const char* model_path;
+ float scale;
+ int target_width;
+ int target_height;
+ int steps;
+ float denoising_strength;
+ int upscale_tile_size;
+ float* custom_sigmas;
+ int custom_sigmas_count;
+
+ HiresConfig();
+ };
+
+ struct ContextConfig {
+ const char* model_path{};
+ const char* clip_l_path{};
+ const char* clip_g_path{};
+ const char* clip_vision_path{};
+ const char* t5xxl_path{};
+ const char* llm_path{};
+ const char* llm_vision_path{};
+ const char* diffusion_model_path{};
+ const char* high_noise_diffusion_model_path{};
+ const char* uncond_diffusion_model_path{};
+ const char* embeddings_connectors_path{};
+ const char* vae_path{};
+ const char* audio_vae_path{};
+ const char* audio_encoder_path{};
+ const char* taesd_path{};
+ const char* control_net_path{};
+ const char* ip_adapter_path{};
+ const char* motion_module_path{};
+ const sd_embedding_t* embeddings{};
+ uint32_t embedding_count{};
+ const char* photo_maker_path{};
+ const char* pulid_weights_path{};
+ const char* tensor_type_rules{};
+ int n_threads{};
+ enum sd_type_t wtype {};
+ enum rng_type_t rng_type {};
+ enum rng_type_t sampler_rng_type {};
+ enum prediction_t prediction {};
+ enum lora_apply_mode_t lora_apply_mode {};
+ bool enable_mmap{};
+ bool flash_attn{};
+ bool diffusion_flash_attn{};
+ bool tae_preview_only{};
+ bool diffusion_conv_direct{};
+ bool vae_conv_direct{};
+ bool force_sdxl_vae_conv_scale{};
+ enum sd_vae_format_t vae_format {};
+ const char* max_vram{}; // Optional per-device GiB budget for managed weights and runner buffers; 0 uses live free VRAM without an explicit budget
+ bool disable_prefetch{}; // Disable asynchronous next-segment weight prefetch
+ bool eager_load{}; // Load all params into the params backend at model-load time instead of lazily on first use
+ const char* backend{};
+ const char* params_backend{};
+ const char* split_mode{}; // weight distribution for multi-device modules: layer (default) or row, or per-module assignments e.g. "diffusion=row"
+ bool auto_fit{};
+ const char* rpc_servers{};
+ const char* model_args{};
+ bool disable_segmented_compute{}; // Force monolithic graph execution even when automatic graph cutting would fit memory better
+ bool batched_cfg{}; // Run the conditional and unconditional CFG branches in one batched UNet forward when supported
+ float linear_scale{}; // Override linear input scaling; 0 keeps the model default
+ float attn_scale{}; // Override flash-attention K/V scaling; 0 keeps the model default
+ const char* tokenizer{}; // tokenizer.json path or main=FILE,clip-l=FILE,clip-g=FILE assignments; required for PiD and Lens
+ bool sage_attn{};
+ int conditioning_cache_size{}; // Maximum cached conditioning entries per context; 0 disables caching (default: 4)
+ const char* flash_attn_mode{}; // on/off/auto or module assignments (e.g. "auto,te=off"); NULL/empty honors legacy flags.
+ const char* conv_direct_mode{}; // on/off/auto or diffusion/vae/controlnet assignments; NULL/empty honors legacy flags.
+
+ ContextConfig();
+ };
+
+ struct ImageConfig {
+ const sd_lora_t* loras{};
+ uint32_t lora_count{};
+ const char* prompt{};
+ const char* negative_prompt{};
+ int clip_skip{};
+ sd::Image init_image{};
+ sd::Image* ref_images{};
+ int ref_images_count{};
+ const char* ref_image_args{};
+ sd::Image mask_image{};
+ int width{};
+ int height{};
+ sd::SampleConfig sample_params{};
+ float strength{};
+ int64_t seed{};
+ int batch_count{};
+ sd::Image control_image{};
+ float control_strength{};
+ sd::Image ip_adapter_image{};
+ float ip_adapter_strength{};
+ sd::PhotoMakerConfig pm_params{};
+ sd::PuLIDConfig pulid_params{};
+ sd::TilingConfig vae_tiling_params{};
+ sd::CacheConfig cache{};
+ sd::HiresConfig hires{};
+ int qwen_image_layers{};
+ bool circular_x{};
+ bool circular_y{};
+ sd::ImagePreprocessConfig image_preprocess{};
+ enum sd_control_type_t control_type {};
+
+ ImageConfig();
+ };
+
+ struct VideoConfig {
+ const sd_lora_t* loras{};
+ uint32_t lora_count{};
+ const char* prompt{};
+ const char* negative_prompt{};
+ int clip_skip{};
+ sd::Image init_image{};
+ sd::Image end_image{};
+ sd::Image* ref_images{};
+ int ref_images_count{};
+ sd::ReferenceVideo* ref_videos{};
+ int ref_videos_count{};
+ sd::Audio* ref_audios{};
+ int ref_audios_count{};
+ sd::Image* control_frames{};
+ int control_frames_size{};
+ int width{};
+ int height{};
+ sd::SampleConfig sample_params{};
+ sd::SampleConfig high_noise_sample_params{};
+ float moe_boundary{};
+ float strength{};
+ int64_t seed{};
+ int video_frames{};
+ int fps{};
+ float vace_strength{};
+ sd::TilingConfig vae_tiling_params{};
+ sd::CacheConfig cache{};
+ sd::HiresConfig hires{};
+ bool circular_x{};
+ bool circular_y{};
+ sd::ImagePreprocessConfig image_preprocess{};
+ const char* ref_image_args{};
+
+ VideoConfig();
+ };
+
+ struct VideoUpscalerConfig {
+ const char* diffusion_model_path{};
+ const char* vae_path{};
+ // Explicit output scale; RelayVSR requires 2 or 4 matching both checkpoints.
+ int upscale_factor{};
+ int n_threads{};
+ const char* backend{};
+ const char* params_backend{};
+ // auto/on/off or diffusion/vae assignments; null selects auto.
+ const char* flash_attn_mode{};
+ const char* conv_direct_mode{};
+ };
+
+ struct VideoUpscaleConfig {
+ const sd::Image* frames{};
+ int frame_count{};
+ int fps = 24;
+ int64_t seed = 42;
+ // RelayVSR reference window, including both endpoints; defaults to 16.
+ int window_size = 16;
+ // SeedVR2 independent temporal chunks in input frames (1 or 4n+1); defaults to 5.
+ int temporal_chunk_size = 5;
+ };
+
+ struct ADetailConfig {
+ const char* prompt{};
+ const char* negative_prompt{};
+ const char* extra_ad_args{};
+ };
+
+ struct PostprocessConfig {
+ sd::Image image{};
+ bool conv_direct = false;
+ int tile_size = 128;
+ VideoUpscalerConfig video_upscaler;
+ VideoUpscaleConfig video_upscale;
+ ADetailConfig adetail;
+ };
+
+ struct OptConfig {
+ ContextConfig context;
+ ImageConfig image;
+ VideoConfig video;
+ PostprocessConfig post;
+ };
+
+} // namespace sd
+
+#endif // SD_CONFIG_H
diff --git a/src/core/backend_fit.cpp b/src/core/backend_fit.cpp
index 120cbe4..123894d 100644
--- a/src/core/backend_fit.cpp
+++ b/src/core/backend_fit.cpp
@@ -1,4 +1,5 @@
#include "backend_fit.h"
+#include "config.h"
#include <algorithm>
#include <cctype>
@@ -489,7 +490,7 @@ namespace sd::backend_fit {
return true;
}
- bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params,
+ bool prepare_vae_decode_retry_tiling(sd::TilingConfig& tiling_params,
bool prefer_temporal_tiling,
ggml_status status,
int latent_tile_size_w,
diff --git a/src/core/backend_fit.h b/src/core/backend_fit.h
index ad1b60c..61393ad 100644
--- a/src/core/backend_fit.h
+++ b/src/core/backend_fit.h
@@ -3,6 +3,7 @@
#include <string>
+#include "config.h"
#include "core/ggml_graph_cut.h"
#include "model_loader.h"
#include "stable-diffusion.h"
@@ -15,7 +16,7 @@ namespace sd::backend_fit {
std::string& runtime_spec,
std::string& params_spec);
- bool prepare_vae_decode_retry_tiling(sd_tiling_params_t& tiling_params,
+ bool prepare_vae_decode_retry_tiling(sd::TilingConfig& tiling_params,
bool prefer_temporal_tiling,
ggml_status status,
int latent_tile_size_w,
diff --git a/src/core/cuda/int8.cu b/src/core/cuda/int8.cu
index 1b1ee67..46afe43 100644
--- a/src/core/cuda/int8.cu
+++ b/src/core/cuda/int8.cu
@@ -339,9 +339,9 @@ namespace SD_CUDA_NAMESPACE {
} else {
rotated = values[indices[i]];
}
- int value = row_scale == 0.0f ? 0 : __float2int_rn(rotated * inv_scale);
- value = max(-127, min(127, value));
- output[indices[i]] = (int8_t)value;
+ int value = row_scale == 0.0f ? 0 : __float2int_rn(rotated * inv_scale);
+ value = max(-127, min(127, value));
+ output[indices[i]] = (int8_t)value;
}
if (group == 0 && tid == 0 && dst_stride != k) {
*((float*)(dst + row * dst_stride + k)) = row_scale;
diff --git a/src/core/ggml_tensor_utils.cpp b/src/core/ggml_tensor_utils.cpp
index 5639c7e..44846c6 100644
--- a/src/core/ggml_tensor_utils.cpp
+++ b/src/core/ggml_tensor_utils.cpp
@@ -200,7 +200,7 @@ uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, int idx, bool video) {
return image_data;
}
-void sd_image_to_ggml_tensor(sd_image_t image,
+void sd_image_to_ggml_tensor(sd::Image image,
ggml_tensor* tensor,
bool scale) {
GGML_ASSERT(image.width == tensor->ne[0]);
diff --git a/src/core/ggml_tensor_utils.h b/src/core/ggml_tensor_utils.h
index 65cc5f2..f7b101c 100644
--- a/src/core/ggml_tensor_utils.h
+++ b/src/core/ggml_tensor_utils.h
@@ -1,6 +1,8 @@
#ifndef __SD_CORE_GGML_TENSOR_UTILS_H__
#define __SD_CORE_GGML_TENSOR_UTILS_H__
+#include "core/media.h"
+
#include <cmath>
#include <cstdint>
#include <cstdio>
@@ -58,7 +60,7 @@ __STATIC_INLINE__ ggml_fp16_t ggml_ext_tensor_get_f16(const ggml_tensor* tensor,
return *(ggml_fp16_t*)((char*)(tensor->data) + i3 * tensor->nb[3] + i2 * tensor->nb[2] + i1 * tensor->nb[1] + i0 * tensor->nb[0]);
}
-__STATIC_INLINE__ float sd_image_get_f32(sd_image_t image, int64_t iw, int64_t ih, int64_t ic, bool scale = true) {
+__STATIC_INLINE__ float sd_image_get_f32(sd::Image image, int64_t iw, int64_t ih, int64_t ic, bool scale = true) {
float value = *(image.data + ih * image.width * image.channel + iw * image.channel + ic);
if (scale) {
value /= 255.f;
@@ -146,7 +148,7 @@ uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, uint8_t* image_data = nullp
uint8_t* ggml_tensor_to_sd_image(ggml_tensor* input, int idx, bool video = false);
-void sd_image_to_ggml_tensor(sd_image_t image,
+void sd_image_to_ggml_tensor(sd::Image image,
ggml_tensor* tensor,
bool scale = true);
diff --git a/src/core/media.h b/src/core/media.h
new file mode 100644
index 0000000..3ee1d4e
--- /dev/null
+++ b/src/core/media.h
@@ -0,0 +1,33 @@
+#ifndef __SD_CORE_MEDIA_H__
+#define __SD_CORE_MEDIA_H__
+
+#include <cstdint>
+
+namespace sd {
+ struct Image {
+ uint32_t width;
+ uint32_t height;
+ uint32_t channel;
+ uint8_t* data;
+ };
+
+ struct Audio {
+ uint32_t sample_rate;
+ uint32_t channels;
+ uint64_t sample_count;
+ float* data;
+ };
+
+ struct ReferenceVideo {
+ Image* frames;
+ int frame_count;
+ int fps;
+ Audio audio;
+ };
+
+ bool preprocess_canny(Image image, float high_threshold, float low_threshold, float weak, float strong, bool inverse);
+ void free_image(Image* image);
+ void free_audio(Audio* audio);
+}
+
+#endif // __SD_CORE_MEDIA_H__
diff --git a/src/core/option_types.cpp b/src/core/option_types.cpp
new file mode 100644
index 0000000..2c9a1c1
--- /dev/null
+++ b/src/core/option_types.cpp
@@ -0,0 +1,265 @@
+#include "option_types.h"
+
+#include <cstring>
+
+static constexpr const char* rng_type_to_str[] = {
+ "std_default",
+ "cuda",
+ "cpu",
+};
+
+const char* sd_rng_type_name(enum rng_type_t rng_type) {
+ if (rng_type < RNG_TYPE_COUNT) {
+ return rng_type_to_str[rng_type];
+ }
+ return "NONE";
+}
+
+enum rng_type_t str_to_rng_type(const char* str) {
+ for (int i = 0; i < RNG_TYPE_COUNT; i++) {
+ if (!strcmp(str, rng_type_to_str[i])) {
+ return (enum rng_type_t)i;
+ }
+ }
+ return RNG_TYPE_COUNT;
+}
+
+static constexpr const char* sample_method_to_str[] = {
+ "euler",
+ "euler_a",
+ "heun",
+ "dpm2",
+ "dpm++2s_a",
+ "dpm++2m",
+ "dpm++2mv2",
+ "ipndm",
+ "ipndm_v",
+ "lcm",
+ "ddim_trailing",
+ "tcd",
+ "res_multistep",
+ "res_2s",
+ "er_sde",
+ "euler_cfg_pp",
+ "euler_a_cfg_pp",
+ "euler_ge",
+ "dpm++2m_sde",
+ "dpm++2m_sde_bt",
+ "lms",
+};
+
+static_assert(SAMPLE_METHOD_COUNT == sizeof(sample_method_to_str) / sizeof(sample_method_to_str[0]),
+ "\nnumber of elements in sample_method_to_str[] != SAMPLE_METHOD_COUNT");
+
+const char* sd_sample_method_name(enum sample_method_t sample_method) {
+ if (sample_method < SAMPLE_METHOD_COUNT) {
+ return sample_method_to_str[sample_method];
+ }
+ return "NONE";
+}
+
+enum sample_method_t str_to_sample_method(const char* str) {
+ for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) {
+ if (!strcmp(str, sample_method_to_str[i])) {
+ return (enum sample_method_t)i;
+ }
+ }
+ return SAMPLE_METHOD_COUNT;
+}
+
+static constexpr const char* scheduler_to_str[] = {
+ "discrete",
+ "karras",
+ "exponential",
+ "ays",
+ "gits",
+ "sgm_uniform",
+ "simple",
+ "smoothstep",
+ "kl_optimal",
+ "lcm",
+ "bong_tangent",
+ "ltx2",
+ "logit_normal",
+ "flux2",
+ "flux",
+ "beta",
+ "llada_image",
+};
+
+static_assert(SCHEDULER_COUNT == sizeof(scheduler_to_str) / sizeof(scheduler_to_str[0]),
+ "\nnumber of elements in scheduler_to_str[] != SCHEDULER_COUNT");
+
+const char* sd_scheduler_name(enum scheduler_t scheduler) {
+ if (scheduler < SCHEDULER_COUNT) {
+ return scheduler_to_str[scheduler];
+ }
+ return "NONE";
+}
+
+enum scheduler_t str_to_scheduler(const char* str) {
+ if (!strcmp(str, "normal")) {
+ return DISCRETE_SCHEDULER;
+ }
+ for (int i = 0; i < SCHEDULER_COUNT; i++) {
+ if (!strcmp(str, scheduler_to_str[i])) {
+ return (enum scheduler_t)i;
+ }
+ }
+ return SCHEDULER_COUNT;
+}
+
+static constexpr const char* prediction_to_str[] = {
+ "eps",
+ "v",
+ "edm_v",
+ "sd3_flow",
+ "flux_flow",
+ "sefi_flow",
+ "minit2i_flow",
+ "sensenova_u1_flow",
+ "flux3_flow",
+ "h3_av_flow",
+ "ming_image_flow",
+};
+
+const char* sd_prediction_name(enum prediction_t prediction) {
+ if (prediction >= EPS_PRED && prediction < PREDICTION_COUNT) {
+ return prediction_to_str[prediction];
+ }
+ return "NONE";
+}
+
+enum prediction_t str_to_prediction(const char* str) {
+ for (int i = 0; i < PREDICTION_COUNT; i++) {
+ if (!strcmp(str, prediction_to_str[i])) {
+ return (enum prediction_t)i;
+ }
+ }
+ return PREDICTION_COUNT;
+}
+
+static constexpr const char* preview_to_str[] = {
+ "none",
+ "proj",
+ "tae",
+ "vae",
+};
+
+const char* sd_preview_name(enum preview_t preview) {
+ if (preview < PREVIEW_COUNT) {
+ return preview_to_str[preview];
+ }
+ return "NONE";
+}
+
+enum preview_t str_to_preview(const char* str) {
+ for (int i = 0; i < PREVIEW_COUNT; i++) {
+ if (!strcmp(str, preview_to_str[i])) {
+ return (enum preview_t)i;
+ }
+ }
+ return PREVIEW_COUNT;
+}
+
+static constexpr const char* lora_apply_mode_to_str[] = {
+ "auto",
+ "immediately",
+ "at_runtime",
+};
+
+const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode) {
+ if (mode < LORA_APPLY_MODE_COUNT) {
+ return lora_apply_mode_to_str[mode];
+ }
+ return "NONE";
+}
+
+enum lora_apply_mode_t str_to_lora_apply_mode(const char* str) {
+ for (int i = 0; i < LORA_APPLY_MODE_COUNT; i++) {
+ if (!strcmp(str, lora_apply_mode_to_str[i])) {
+ return (enum lora_apply_mode_t)i;
+ }
+ }
+ return LORA_APPLY_MODE_COUNT;
+}
+
+static constexpr const char* hires_upscaler_to_str[] = {
+ "None",
+ "Latent",
+ "Latent (nearest)",
+ "Latent (nearest-exact)",
+ "Latent (antialiased)",
+ "Latent (bicubic)",
+ "Latent (bicubic antialiased)",
+ "Lanczos",
+ "Nearest",
+ "Model",
+};
+
+const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler) {
+ if (upscaler >= SD_HIRES_UPSCALER_NONE && upscaler < SD_HIRES_UPSCALER_COUNT) {
+ return hires_upscaler_to_str[upscaler];
+ }
+ return "NONE";
+}
+
+enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str) {
+ for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; i++) {
+ if (!strcmp(str, hires_upscaler_to_str[i])) {
+ return (enum sd_hires_upscaler_t)i;
+ }
+ }
+ return SD_HIRES_UPSCALER_COUNT;
+}
+
+static constexpr const char* control_type_to_str[] = {
+ "none",
+ "openpose",
+ "depth",
+ "scribble",
+ "canny",
+ "normal",
+ "segment",
+ "tile",
+ "repaint",
+};
+
+static_assert(SD_CONTROL_TYPE_COUNT == sizeof(control_type_to_str) / sizeof(control_type_to_str[0]),
+ "\nnumber of elements in control_type_to_str[] != SD_CONTROL_TYPE_COUNT");
+
+const char* sd_control_type_name(enum sd_control_type_t control_type) {
+ if (control_type >= SD_CONTROL_TYPE_NONE && control_type < SD_CONTROL_TYPE_COUNT) {
+ return control_type_to_str[control_type];
+ }
+ return "NONE";
+}
+
+enum sd_control_type_t str_to_sd_control_type(const char* str) {
+ for (int i = 0; i < SD_CONTROL_TYPE_COUNT; i++) {
+ if (!strcmp(str, control_type_to_str[i])) {
+ return (enum sd_control_type_t)i;
+ }
+ }
+ static const struct {
+ const char* alias;
+ enum sd_control_type_t type;
+ } aliases[] = {
+ {"hed", SD_CONTROL_TYPE_SCRIBBLE},
+ {"softedge", SD_CONTROL_TYPE_SCRIBBLE},
+ {"pidi", SD_CONTROL_TYPE_SCRIBBLE},
+ {"ted", SD_CONTROL_TYPE_SCRIBBLE},
+ {"lineart", SD_CONTROL_TYPE_CANNY},
+ {"anime_lineart", SD_CONTROL_TYPE_CANNY},
+ {"mlsd", SD_CONTROL_TYPE_CANNY},
+ {"seg", SD_CONTROL_TYPE_SEGMENT},
+ {"inpaint", SD_CONTROL_TYPE_REPAINT},
+ {"outpaint", SD_CONTROL_TYPE_REPAINT},
+ };
+ for (const auto& entry : aliases) {
+ if (!strcmp(str, entry.alias)) {
+ return entry.type;
+ }
+ }
+ return SD_CONTROL_TYPE_COUNT;
+}
diff --git a/src/core/option_types.h b/src/core/option_types.h
new file mode 100644
index 0000000..280cde2
--- /dev/null
+++ b/src/core/option_types.h
@@ -0,0 +1,165 @@
+#ifndef __SD_CORE_OPTION_TYPES_H__
+#define __SD_CORE_OPTION_TYPES_H__
+
+#include "stable-diffusion.h"
+
+enum rng_type_t {
+ STD_DEFAULT_RNG,
+ CUDA_RNG,
+ CPU_RNG,
+ RNG_TYPE_COUNT
+};
+
+enum sample_method_t {
+ EULER_SAMPLE_METHOD,
+ EULER_A_SAMPLE_METHOD,
+ HEUN_SAMPLE_METHOD,
+ DPM2_SAMPLE_METHOD,
+ DPMPP2S_A_SAMPLE_METHOD,
+ DPMPP2M_SAMPLE_METHOD,
+ DPMPP2Mv2_SAMPLE_METHOD,
+ IPNDM_SAMPLE_METHOD,
+ IPNDM_V_SAMPLE_METHOD,
+ LCM_SAMPLE_METHOD,
+ DDIM_TRAILING_SAMPLE_METHOD,
+ TCD_SAMPLE_METHOD,
+ RES_MULTISTEP_SAMPLE_METHOD,
+ RES_2S_SAMPLE_METHOD,
+ ER_SDE_SAMPLE_METHOD,
+ EULER_CFG_PP_SAMPLE_METHOD,
+ EULER_A_CFG_PP_SAMPLE_METHOD,
+ EULER_GE_SAMPLE_METHOD,
+ DPMPP2M_SDE_SAMPLE_METHOD,
+ DPMPP2M_SDE_BT_SAMPLE_METHOD,
+ LMS_SAMPLE_METHOD,
+ SAMPLE_METHOD_COUNT
+};
+
+enum scheduler_t {
+ DISCRETE_SCHEDULER,
+ KARRAS_SCHEDULER,
+ EXPONENTIAL_SCHEDULER,
+ AYS_SCHEDULER,
+ GITS_SCHEDULER,
+ SGM_UNIFORM_SCHEDULER,
+ SIMPLE_SCHEDULER,
+ SMOOTHSTEP_SCHEDULER,
+ KL_OPTIMAL_SCHEDULER,
+ LCM_SCHEDULER,
+ BONG_TANGENT_SCHEDULER,
+ LTX2_SCHEDULER,
+ LOGIT_NORMAL_SCHEDULER,
+ FLUX2_SCHEDULER,
+ FLUX_SCHEDULER,
+ BETA_SCHEDULER,
+ LLADA_IMAGE_SCHEDULER,
+ SCHEDULER_COUNT
+};
+
+enum prediction_t {
+ EPS_PRED,
+ V_PRED,
+ EDM_V_PRED,
+ FLOW_PRED,
+ FLUX_FLOW_PRED,
+ SEFI_FLOW_PRED,
+ MINIT2I_FLOW_PRED,
+ SENSENOVA_U1_FLOW_PRED,
+ FLUX3_FLOW_PRED,
+ H3_AV_FLOW_PRED,
+ MING_IMAGE_FLOW_PRED,
+ PREDICTION_COUNT
+};
+
+enum lora_apply_mode_t {
+ LORA_APPLY_AUTO,
+ LORA_APPLY_IMMEDIATELY,
+ LORA_APPLY_AT_RUNTIME,
+ LORA_APPLY_MODE_COUNT,
+};
+
+enum sd_vae_format_t {
+ SD_VAE_FORMAT_AUTO = -1,
+ SD_VAE_FORMAT_FLUX,
+ SD_VAE_FORMAT_SD3,
+ SD_VAE_FORMAT_FLUX2,
+ SD_VAE_FORMAT_WAN,
+ SD_VAE_FORMAT_COUNT,
+};
+
+enum sd_cache_mode_t {
+ SD_CACHE_DISABLED = 0,
+ SD_CACHE_EASYCACHE,
+ SD_CACHE_UCACHE,
+ SD_CACHE_DBCACHE,
+ SD_CACHE_TAYLORSEER,
+ SD_CACHE_CACHE_DIT,
+ SD_CACHE_SPECTRUM,
+};
+
+enum sd_hires_upscaler_t {
+ SD_HIRES_UPSCALER_NONE,
+ SD_HIRES_UPSCALER_LATENT,
+ SD_HIRES_UPSCALER_LATENT_NEAREST,
+ SD_HIRES_UPSCALER_LATENT_NEAREST_EXACT,
+ SD_HIRES_UPSCALER_LATENT_ANTIALIASED,
+ SD_HIRES_UPSCALER_LATENT_BICUBIC,
+ SD_HIRES_UPSCALER_LATENT_BICUBIC_ANTIALIASED,
+ SD_HIRES_UPSCALER_LANCZOS,
+ SD_HIRES_UPSCALER_NEAREST,
+ SD_HIRES_UPSCALER_MODEL,
+ SD_HIRES_UPSCALER_COUNT,
+};
+
+enum sd_control_type_t {
+ SD_CONTROL_TYPE_NONE,
+ SD_CONTROL_TYPE_OPENPOSE,
+ SD_CONTROL_TYPE_DEPTH,
+ SD_CONTROL_TYPE_SCRIBBLE, // thick lines: scribble, hed, softedge, ted
+ SD_CONTROL_TYPE_CANNY, // thin lines: canny, lineart, anime lineart, mlsd
+ SD_CONTROL_TYPE_NORMAL,
+ SD_CONTROL_TYPE_SEGMENT,
+ SD_CONTROL_TYPE_TILE,
+ SD_CONTROL_TYPE_REPAINT, // inpaint, outpaint
+ SD_CONTROL_TYPE_COUNT,
+};
+
+const char* sd_rng_type_name(enum rng_type_t rng_type);
+
+enum rng_type_t str_to_rng_type(const char* str);
+
+const char* sd_sample_method_name(enum sample_method_t sample_method);
+
+enum sample_method_t str_to_sample_method(const char* str);
+
+const char* sd_scheduler_name(enum scheduler_t scheduler);
+
+enum scheduler_t str_to_scheduler(const char* str);
+
+const char* sd_prediction_name(enum prediction_t prediction);
+
+enum prediction_t str_to_prediction(const char* str);
+
+const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode);
+
+enum lora_apply_mode_t str_to_lora_apply_mode(const char* str);
+
+const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler);
+
+enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str);
+
+const char* sd_control_type_name(enum sd_control_type_t control_type);
+
+enum sd_control_type_t str_to_sd_control_type(const char* str);
+
+enum preview_t {
+ PREVIEW_NONE,
+ PREVIEW_PROJ,
+ PREVIEW_TAE,
+ PREVIEW_VAE,
+ PREVIEW_COUNT
+};
+const char* sd_preview_name(enum preview_t preview);
+enum preview_t str_to_preview(const char* str);
+
+#endif // __SD_CORE_OPTION_TYPES_H__
diff --git a/src/core/util.cpp b/src/core/util.cpp
index 9de1ace..220134f 100644
--- a/src/core/util.cpp
+++ b/src/core/util.cpp
@@ -17,6 +17,7 @@
#include <unordered_set>
#include <vector>
#include "core/ggml_tensor_utils.h"
+#include "core/option_types.h"
#include "runtime/preprocessing.hpp"
#if defined(__APPLE__) && defined(__MACH__)
@@ -721,7 +722,7 @@ void sd_set_progress_callback(sd_progress_cb_t cb, void* data) {
sd_progress_cb = cb;
sd_progress_cb_data = data;
}
-void sd_set_preview_callback(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data) {
+void sd_set_preview_callback_impl(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data) {
sd_preview_info = {};
sd_preview_cb = cb;
sd_preview_cb_data = data;
@@ -782,7 +783,7 @@ void* sd_get_progress_callback_data() {
return sd_progress_cb_data;
}
-sd_image_t tensor_to_sd_image(const sd::Tensor<float>& tensor, int frame_index) {
+sd::Image tensor_to_sd_image(const sd::Tensor<float>& tensor, int frame_index) {
const auto& shape = tensor.shape();
GGML_ASSERT(shape.size() == 4 || shape.size() == 5);
int width = static_cast<int>(shape[0]);
@@ -799,7 +800,7 @@ sd_image_t tensor_to_sd_image(const sd::Tensor<float>& tensor, int frame_index)
};
}
-sd::Tensor<float> sd_image_to_tensor(sd_image_t image,
+sd::Tensor<float> sd_image_to_tensor(sd::Image image,
int target_width,
int target_height,
bool scale) {
diff --git a/src/core/util.h b/src/core/util.h
index 0929f60..082565a 100644
--- a/src/core/util.h
+++ b/src/core/util.h
@@ -1,12 +1,15 @@
#ifndef __SD_CORE_UTIL_H__
#define __SD_CORE_UTIL_H__
+#include "core/media.h"
+
#include <cstdint>
#include <memory>
#include <string>
#include <utility>
#include <vector>
+#include "core/option_types.h"
#include "core/tensor.hpp"
#include "ggml-backend.h"
#include "stable-diffusion.h"
@@ -41,9 +44,9 @@ std::string utf32_to_utf8(const std::u32string& utf32_str);
std::u32string unicode_value_to_utf32(int unicode_value);
// std::string sd_basename(const std::string& path);
-sd_image_t tensor_to_sd_image(const sd::Tensor<float>& tensor, int frame_index = 0);
+sd::Image tensor_to_sd_image(const sd::Tensor<float>& tensor, int frame_index = 0);
-sd::Tensor<float> sd_image_to_tensor(sd_image_t image,
+sd::Tensor<float> sd_image_to_tensor(sd::Image image,
int target_width = -1,
int target_height = -1,
bool scale = true);
@@ -105,6 +108,7 @@ void* sd_get_progress_callback_data();
void sd_begin_preview_pass(int total_steps);
sd_preview_cb_t sd_get_preview_callback();
void* sd_get_preview_callback_data();
+void sd_set_preview_callback_impl(sd_preview_cb_t cb, preview_t mode, int interval, bool denoised, bool noisy, void* data);
preview_t sd_get_preview_mode();
int sd_get_preview_interval();
bool sd_should_preview_denoised();
diff --git a/src/detailer.cpp b/src/detailer.cpp
index 9c48ca4..234afce 100644
--- a/src/detailer.cpp
+++ b/src/detailer.cpp
@@ -1,5 +1,9 @@
#include "detailer.h"
+#include "api/options.h"
+#include "api/result.h"
+#include "core/option_types.h"
#include "generation_info.h"
+#include "pipeline/generation.h"
#include "pipeline/request.h"
#include <algorithm>
@@ -15,8 +19,8 @@
#include "json.hpp"
#include "model_loader.h"
-struct adetailer_ctx_t {
- ADetailerGGML* detailer = nullptr;
+struct sd_adetailer {
+ std::unique_ptr<ADetailerGGML> detailer;
};
struct LetterboxInput {
@@ -96,7 +100,7 @@ static bool parse_float(const std::string& value, float* result) {
}
}
-static float image_channel(const sd_image_t& image, int x, int y, int channel) {
+static float image_channel(const sd::Image& image, int x, int y, int channel) {
if (image.data == nullptr || image.width == 0 || image.height == 0 || image.channel == 0) {
return 0.f;
}
@@ -107,7 +111,7 @@ static float image_channel(const sd_image_t& image, int x, int y, int channel) {
return image.data[index] / 255.f;
}
-static float bilinear_channel(const sd_image_t& image, float x, float y, int channel) {
+static float bilinear_channel(const sd::Image& image, float x, float y, int channel) {
int x0 = static_cast<int>(std::floor(x));
int y0 = static_cast<int>(std::floor(y));
int x1 = x0 + 1;
@@ -119,7 +123,7 @@ static float bilinear_channel(const sd_image_t& image, float x, float y, int cha
return a * (1.f - wy) + b * wy;
}
-static LetterboxInput make_letterbox_input(sd_image_t image, int input_size) {
+static LetterboxInput make_letterbox_input(sd::Image image, int input_size) {
LetterboxInput result;
result.tensor = sd::full<float>({input_size, input_size, 3, 1}, 114.f / 255.f);
result.scale = std::min(static_cast<float>(input_size) / image.width,
@@ -180,7 +184,7 @@ static float box_iou(const ADetailerDetection& a, const ADetailerDetection& b) {
static std::vector<ADetailerDetection> decode_detections(const sd::Tensor<float>& raw,
const YOLOv8Config& config,
const LetterboxInput& letterbox,
- sd_image_t image,
+ sd::Image image,
const ADetailerParams& params) {
std::vector<ADetailerDetection> candidates;
if (raw.empty() || raw.dim() < 2) {
@@ -510,9 +514,9 @@ static CropRegion expand_crop(CropRegion crop,
return crop;
}
-static sd_image_t resize_crop_image(const sd_image_t& source, const CropRegion& crop, int width, int height) {
- sd_image_t output = {static_cast<uint32_t>(width), static_cast<uint32_t>(height), 3, nullptr};
- output.data = static_cast<uint8_t*>(malloc(static_cast<size_t>(width) * height * 3));
+static sd::Image resize_crop_image(const sd::Image& source, const CropRegion& crop, int width, int height) {
+ sd::Image output = {static_cast<uint32_t>(width), static_cast<uint32_t>(height), 3, nullptr};
+ output.data = static_cast<uint8_t*>(malloc(static_cast<size_t>(width) * height * 3));
if (output.data == nullptr) {
return {0, 0, 0, nullptr};
}
@@ -531,9 +535,9 @@ static sd_image_t resize_crop_image(const sd_image_t& source, const CropRegion&
return output;
}
-static sd_image_t resize_crop_mask(const Mask& source, const CropRegion& crop, int width, int height) {
- sd_image_t output = {static_cast<uint32_t>(width), static_cast<uint32_t>(height), 1, nullptr};
- output.data = static_cast<uint8_t*>(malloc(static_cast<size_t>(width) * height));
+static sd::Image resize_crop_mask(const Mask& source, const CropRegion& crop, int width, int height) {
+ sd::Image output = {static_cast<uint32_t>(width), static_cast<uint32_t>(height), 1, nullptr};
+ output.data = static_cast<uint8_t*>(malloc(static_cast<size_t>(width) * height));
if (output.data == nullptr) {
return {0, 0, 0, nullptr};
}
@@ -549,11 +553,11 @@ static sd_image_t resize_crop_mask(const Mask& source, const CropRegion& crop, i
return output;
}
-static sd_image_t copy_for_detailing(const sd_image_t& source) {
+static sd::Image copy_for_detailing(const sd::Image& source) {
if (source.channel == 4) {
- sd_image_t output = source;
- size_t bytes = static_cast<size_t>(source.width) * source.height * source.channel;
- output.data = static_cast<uint8_t*>(malloc(bytes));
+ sd::Image output = source;
+ size_t bytes = static_cast<size_t>(source.width) * source.height * source.channel;
+ output.data = static_cast<uint8_t*>(malloc(bytes));
if (output.data != nullptr) {
memcpy(output.data, source.data, bytes);
}
@@ -563,8 +567,8 @@ static sd_image_t copy_for_detailing(const sd_image_t& source) {
return resize_crop_image(source, full, source.width, source.height);
}
-static void composite_crop(sd_image_t* destination,
- const sd_image_t& generated,
+static void composite_crop(sd::Image* destination,
+ const sd::Image& generated,
const Mask& feather_mask,
const CropRegion& crop) {
int crop_width = crop.x2 - crop.x1;
@@ -724,7 +728,7 @@ bool ADetailerGGML::load_from_file(const std::string& detector_path) {
return true;
}
-std::vector<ADetailerDetection> ADetailerGGML::predict(sd_image_t image,
+std::vector<ADetailerDetection> ADetailerGGML::predict(sd::Image image,
const ADetailerParams& params) {
LetterboxInput input = make_letterbox_input(image, params.input_size);
int64_t start = ggml_time_ms();
@@ -758,8 +762,8 @@ std::vector<ADetailerDetection> ADetailerGGML::predict(sd_image_t image,
return detections;
}
-static bool parse_adetailer_params(const sd_adetailer_params_t& public_params,
- const sd_img_gen_params_t& inpaint_params,
+static bool parse_adetailer_params(const sd::ADetailConfig& public_params,
+ const sd::ImageConfig& inpaint_params,
ADetailerParams* params) {
if (params == nullptr) {
return false;
@@ -883,41 +887,32 @@ static bool parse_adetailer_params(const sd_adetailer_params_t& public_params,
return true;
}
-adetailer_ctx_t* new_adetailer_ctx(const char* detector_path,
- int n_threads,
- const char* backend,
- const char* params_backend) {
- if (detector_path == nullptr || detector_path[0] == '\0') {
- return nullptr;
- }
- auto* context = static_cast<adetailer_ctx_t*>(calloc(1, sizeof(adetailer_ctx_t)));
- if (context == nullptr) {
- return nullptr;
- }
- context->detailer = new ADetailerGGML(n_threads, SAFE_STR(backend), SAFE_STR(params_backend));
- if (context->detailer == nullptr || !context->detailer->load_from_file(detector_path)) {
- delete context->detailer;
- free(context);
- return nullptr;
- }
- return context;
+sd_status_t sd_adetailer_create(const sd_adetailer_opt_t* opt, sd_adetailer_t** out) {
+ if (out)
+ *out = nullptr;
+ return sd::api::call([&] {
+ if (!out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(opt, sd::api::ADetailer, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ auto result = std::make_unique<sd_adetailer_t>();
+ const auto& ctx = config.context;
+ result->detailer = std::make_unique<ADetailerGGML>(ctx.n_threads, SAFE_STR(ctx.backend), SAFE_STR(ctx.params_backend));
+ if (!result->detailer->load_from_file(ctx.model_path))
+ return sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "could not load ADetailer");
+ *out = result.release();
+ return SD_STATUS_OK;
+ },
+ SD_STATUS_LOAD_FAILED);
}
-void free_adetailer_ctx(adetailer_ctx_t* context) {
- if (context == nullptr) {
- return;
- }
- delete context->detailer;
- context->detailer = nullptr;
- free(context);
+void sd_adetailer_free(sd_adetailer_t* adetailer) {
+ delete adetailer;
}
-bool adetail_image(adetailer_ctx_t* context,
- sd_ctx_t* sd_ctx,
- sd_image_t input_image,
- const sd_adetailer_params_t* public_params,
- const sd_img_gen_params_t* inpaint_params,
- sd_image_generation_result_t** result_out) try {
+static bool run_adetail(sd_adetailer_t* context, sd_ctx_t* sd_ctx, sd::Image input_image, const sd::ADetailConfig* public_params, const sd::ImageConfig* inpaint_params, sd_result_t** result_out, sd_status_t& failure) {
if (result_out)
*result_out = nullptr;
if (!result_out)
@@ -929,6 +924,7 @@ bool adetail_image(adetailer_ctx_t* context,
ADetailerParams params;
if (!parse_adetailer_params(*public_params, *inpaint_params, ¶ms)) {
+ failure = SD_STATUS_INVALID_ARGUMENT;
return false;
}
@@ -939,8 +935,9 @@ bool adetail_image(adetailer_ctx_t* context,
filter_and_sort_detections(&detections, input_image.width, input_image.height, params);
auto masks = make_masks(detections, input_image.width, input_image.height, params);
- sd_image_t current = copy_for_detailing(input_image);
+ sd::Image current = copy_for_detailing(input_image);
if (current.data == nullptr) {
+ failure = SD_STATUS_OUT_OF_MEMORY;
return false;
}
@@ -959,38 +956,41 @@ bool adetail_image(adetailer_ctx_t* context,
if (!mask_bbox(masks[i], &crop)) {
continue;
}
- crop = params.inpaint_only_masked ? expand_crop(crop,
- current.width,
- current.height,
- params.inpaint_padding,
- params.inpaint_width,
- params.inpaint_height)
- : CropRegion{0, 0, static_cast<int>(current.width), static_cast<int>(current.height)};
- sd_image_t local_image = resize_crop_image(current, crop, params.inpaint_width, params.inpaint_height);
- sd_image_t local_mask = resize_crop_mask(masks[i], crop, params.inpaint_width, params.inpaint_height);
+ crop = params.inpaint_only_masked ? expand_crop(crop,
+ current.width,
+ current.height,
+ params.inpaint_padding,
+ params.inpaint_width,
+ params.inpaint_height)
+ : CropRegion{0, 0, static_cast<int>(current.width), static_cast<int>(current.height)};
+ sd::Image local_image = resize_crop_image(current, crop, params.inpaint_width, params.inpaint_height);
+ sd::Image local_mask = resize_crop_mask(masks[i], crop, params.inpaint_width, params.inpaint_height);
if (local_image.data == nullptr || local_mask.data == nullptr) {
+ failure = SD_STATUS_OUT_OF_MEMORY;
free(local_image.data);
free(local_mask.data);
return false;
}
- sd_img_gen_params_t generation = resolved_inpaint;
- generation.prompt = prompt.c_str();
- generation.negative_prompt = negative_prompt.c_str();
- generation.init_image = local_image;
- generation.mask_image = local_mask;
- generation.width = params.inpaint_width;
- generation.height = params.inpaint_height;
- generation.strength = params.denoising_strength;
- generation.seed = static_cast<int64_t>((static_cast<uint64_t>(resolved_inpaint.seed) + i) & INT64_MAX);
- generation.batch_count = 1;
- generation.ref_images = nullptr;
- generation.ref_images_count = 0;
- generation.control_image = {};
- generation.pm_params = {};
- generation.pulid_params = {};
- generation.hires.enabled = false;
- generation.image_preprocess = {};
+ sd::ImageConfig generation = resolved_inpaint;
+ generation.prompt = prompt.c_str();
+ generation.negative_prompt = negative_prompt.c_str();
+ generation.init_image = local_image;
+ generation.mask_image = local_mask;
+ generation.width = params.inpaint_width;
+ generation.height = params.inpaint_height;
+ generation.strength = params.denoising_strength;
+ generation.seed = static_cast<int64_t>((static_cast<uint64_t>(resolved_inpaint.seed) + i) & INT64_MAX);
+ generation.batch_count = 1;
+ generation.ref_images = nullptr;
+ generation.ref_images_count = 0;
+ generation.control_image = {};
+ generation.pm_params = {};
+ generation.pulid_params = {};
+ generation.hires.enabled = false;
+ generation.image_preprocess = {};
+
+ generation.sample_params.guidance.img_cfg = INFINITY;
if (params.steps > 0) {
generation.sample_params.sample_steps = params.steps;
generation.sample_params.custom_sigmas = nullptr;
@@ -1006,32 +1006,62 @@ bool adetail_image(adetailer_ctx_t* context,
generation.sample_params.scheduler = params.scheduler;
}
- sd_image_generation_result_t* generated = nullptr;
- bool success = generate_image(sd_ctx, &generation, &generated);
+ sd::ImageGenerationResult* generated = nullptr;
+ failure = sd::generate_image(sd_ctx, generation, &generated);
+ std::unique_ptr<sd::ImageGenerationResult, decltype(&sd::free_image_generation_result)> generated_owner(generated, sd::free_image_generation_result);
free(local_image.data);
free(local_mask.data);
- if (!success || generated == nullptr || generated->count <= 0 || generated->items[0].image.data == nullptr) {
- free_sd_image_generation_result(generated);
+ if (failure != SD_STATUS_OK)
+ return false;
+ if (!generated || generated->count <= 0 || !generated->items[0].image.data) {
+ failure = sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "inpainting returned no image");
return false;
}
- std::unique_ptr<sd_image_generation_result_t, decltype(&free_sd_image_generation_result)> generated_owner(generated, &free_sd_image_generation_result);
Mask feather = gaussian_blur_mask(masks[i], params.mask_blur);
composite_crop(¤t, generated->items[0].image, feather, crop);
++applied;
}
const std::vector<nlohmann::json> infos{info};
- auto* outputs = static_cast<sd_image_t*>(calloc(1, sizeof(sd_image_t)));
- if (!outputs)
+ auto* outputs = static_cast<sd::Image*>(calloc(1, sizeof(sd::Image)));
+ if (!outputs) {
+ failure = SD_STATUS_OUT_OF_MEMORY;
return false;
+ }
outputs[0] = current;
current_owner.release();
- if (!sd::make_image_generation_result(outputs, 1, infos, result_out))
+ if (!sd::make_image_result(outputs, 1, infos, result_out)) {
+ failure = SD_STATUS_OUT_OF_MEMORY;
return false;
+ }
LOG_INFO("ADetailer applied %d mask(s)", applied);
return true;
-} catch (const std::exception& e) {
- LOG_ERROR("ADetailer failed: %s", e.what());
- return false;
+}
+
+sd_status_t sd_adetail(sd_adetailer_t* adetailer, sd_ctx_t* ctx, const sd_adetail_opt_t* opt, sd_result_t** out) {
+ if (out)
+ *out = nullptr;
+ return sd::api::call([&] {
+ if (!out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(opt, sd::api::ADetail, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ if (!adetailer || !ctx)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "adetailer and diffusion context are required");
+ if (!sd_ctx_supports_image_generation(ctx))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, nullptr, "inpainting requires an image-generation model");
+ sd_status_t failure = SD_STATUS_GENERATION_FAILED;
+ if (!run_adetail(adetailer, ctx, config.post.image, &config.post.adetail, &config.image, out, failure)) {
+ if (failure == SD_STATUS_OK)
+ failure = SD_STATUS_GENERATION_FAILED;
+ if (*sd_get_last_error())
+ return failure;
+ return sd::api::error(failure, nullptr, "ADetailer failed; see log for details");
+ }
+ return SD_STATUS_OK;
+ },
+ SD_STATUS_GENERATION_FAILED);
}
diff --git a/src/detailer.h b/src/detailer.h
index 0e3ae50..67ede5b 100644
--- a/src/detailer.h
+++ b/src/detailer.h
@@ -1,6 +1,10 @@
#ifndef __SD_DETAILER_H__
#define __SD_DETAILER_H__
+#include "core/media.h"
+
+#include "core/option_types.h"
+
#include <memory>
#include <string>
#include <vector>
@@ -70,7 +74,7 @@ struct ADetailerGGML {
~ADetailerGGML();
bool load_from_file(const std::string& detector_path);
- std::vector<ADetailerDetection> predict(sd_image_t image,
+ std::vector<ADetailerDetection> predict(sd::Image image,
const ADetailerParams& params);
};
diff --git a/src/extensions/generation_extension.h b/src/extensions/generation_extension.h
index e5ffd77..018092e 100644
--- a/src/extensions/generation_extension.h
+++ b/src/extensions/generation_extension.h
@@ -9,6 +9,7 @@
#include <vector>
#include "conditioning/conditioner.hpp"
+#include "config.h"
#include "core/ggml_extend_backend.h"
#include "model/diffusion/model.hpp"
#include "model_loader.h"
@@ -16,7 +17,7 @@
#include "stable-diffusion.h"
struct GenerationExtensionInitContext {
- const sd_ctx_params_t* params;
+ const sd::ContextConfig* params;
SDVersion version;
const String2TensorStorage& tensor_storage_map;
bool photomaker_source_available;
@@ -30,8 +31,8 @@ struct GenerationExtensionInitContext {
struct GenerationExtensionConditionContext {
Conditioner* conditioner;
ConditionerParams& condition_params;
- const sd_pm_params_t& pm_params;
- const sd_pulid_params_t& pulid_params;
+ const sd::PhotoMakerConfig& pm_params;
+ const sd::PuLIDConfig& pulid_params;
int n_threads;
int total_steps;
};
diff --git a/src/generation_info.cpp b/src/generation_info.cpp
index 0d48291..f56ef98 100644
--- a/src/generation_info.cpp
+++ b/src/generation_info.cpp
@@ -1,9 +1,10 @@
#include "generation_info.h"
+#include "config.h"
+#include "core/option_types.h"
#include <cmath>
#include <cstdio>
#include <cstdlib>
-#include <cstring>
#include "pipeline/request.h"
#include "runtime/guidance.h"
@@ -30,7 +31,7 @@ namespace sd {
return std::strtod(buf, nullptr);
}
- static json sampling_info(const sd_sample_params_t& p, bool high_noise = false) {
+ static json sampling_info(const sd::SampleConfig& p, bool high_noise = false) {
const auto& g = p.guidance;
json info = {{"sample_steps", p.sample_steps}, {"guidance", {{"txt_cfg", g.txt_cfg}, {"distilled_guidance", g.distilled_guidance}}}};
if (!high_noise && p.shifted_timestep > 0)
@@ -55,7 +56,7 @@ namespace sd {
return info;
}
- static json hires_info(const sd_hires_params_t& p) {
+ static json hires_info(const sd::HiresConfig& p) {
json info = {{"enabled", p.enabled}};
if (!p.enabled)
return info;
@@ -69,7 +70,7 @@ namespace sd {
return info;
}
- static json cache_info(const sd_cache_params_t& p) {
+ static json cache_info(const sd::CacheConfig& p) {
static const char* modes[] = {"disabled", "easycache", "ucache", "dbcache", "taylorseer", "cache_dit", "spectrum"};
if (p.mode < SD_CACHE_DISABLED || p.mode > SD_CACHE_SPECTRUM)
return {{"mode", static_cast<int>(p.mode)}};
@@ -144,7 +145,7 @@ namespace sd {
return info;
}
- static json image_info(const sd_img_gen_params_t& p) {
+ static json image_info(const sd::ImageConfig& p) {
auto info = common_info(p, "image");
if (p.init_image.data)
info["strength"] = p.strength;
@@ -177,7 +178,7 @@ namespace sd {
return p;
}
- json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params) {
+ json generation_info(const pipeline::GenerationRequest& request, const sd::ImageConfig& params) {
auto p = resolved_params(request, params);
p.qwen_image_layers = request.qwen_image_layers;
auto info = image_info(p);
@@ -188,7 +189,7 @@ namespace sd {
return info;
}
- json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params) {
+ json generation_info(const pipeline::GenerationRequest& request, const sd::VideoConfig& params) {
const auto p = resolved_params(request, params);
auto info = common_info(p, "video");
if (!request.models.empty())
@@ -208,14 +209,13 @@ namespace sd {
return info;
}
- json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params) {
+ json adetailer_generation_info(const sd::ADetailConfig& params, const sd::ImageConfig& inpaint_params) {
auto inpaint = image_info(inpaint_params);
inpaint["strength"] = inpaint_params.strength;
// ADetailer clears these inputs before generating each crop.
for (const auto* key : {"control_strength", "control_type", "pm_params", "pulid_params", "hires", "image_preprocess"})
inpaint.erase(key);
- sd_img_gen_params_t defaults;
- sd_img_gen_params_init(&defaults);
+ sd::ImageConfig defaults;
if (inpaint_params.qwen_image_layers != defaults.qwen_image_layers)
inpaint["qwen_image_layers"] = inpaint_params.qwen_image_layers;
json adetailer = json::object();
@@ -232,69 +232,4 @@ namespace sd {
return {{"schema", "sdcpp.generation/v1"}, {"kind", "image"}, {"upscale", {{"upscale_factor", upscale_factor}}}};
}
- char* copy_generation_info(const json& info) {
- const std::string text = info.dump(-1, ' ', false, json::error_handler_t::replace);
- char* result = static_cast<char*>(malloc(text.size() + 1));
- if (result)
- memcpy(result, text.c_str(), text.size() + 1);
- return result;
- }
-
- bool make_image_generation_result(sd_image_t* images, int count, const std::vector<json>& infos, sd_image_generation_result_t** result_out) {
- if (!result_out || !images || count <= 0 || infos.size() != static_cast<size_t>(count)) {
- free_sd_images(images, count);
- return false;
- }
- auto* result = static_cast<sd_image_generation_result_t*>(calloc(1, sizeof(sd_image_generation_result_t)));
- if (!result) {
- free_sd_images(images, count);
- return false;
- }
- result->items = static_cast<sd_image_result_t*>(calloc(count, sizeof(sd_image_result_t)));
- if (!result->items) {
- free(result);
- free_sd_images(images, count);
- return false;
- }
- result->count = count;
- try {
- for (int i = 0; i < count; ++i) {
- result->items[i].image = images[i];
- images[i] = {};
- result->items[i].generation_info = copy_generation_info(infos[i]);
- if (!result->items[i].generation_info) {
- free_sd_images(images, count);
- free_sd_image_generation_result(result);
- return false;
- }
- }
- } catch (...) {
- free_sd_images(images, count);
- free_sd_image_generation_result(result);
- return false;
- }
- free(images);
- *result_out = result;
- return true;
- }
-}
-
-void free_sd_image_generation_result(sd_image_generation_result_t* result) {
- if (!result)
- return;
- for (int i = 0; i < result->count; ++i) {
- free(result->items[i].image.data);
- free(result->items[i].generation_info);
- }
- free(result->items);
- free(result);
-}
-
-void free_sd_video_generation_result(sd_video_generation_result_t* result) {
- if (!result)
- return;
- free_sd_images(result->frames, result->frame_count);
- free_sd_audio(result->audio);
- free(result->generation_info);
- free(result);
-}
+} // namespace sd
diff --git a/src/generation_info.h b/src/generation_info.h
index 56688ee..d84673f 100644
--- a/src/generation_info.h
+++ b/src/generation_info.h
@@ -3,6 +3,7 @@
#include <vector>
+#include "config.h"
#include "json.hpp"
#include "stable-diffusion.h"
@@ -11,13 +12,11 @@ namespace sd::pipeline {
}
namespace sd {
- nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_img_gen_params_t& params);
- nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd_vid_gen_params_t& params);
- nlohmann::json adetailer_generation_info(const sd_adetailer_params_t& params, const sd_img_gen_params_t& inpaint_params);
+ nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd::ImageConfig& params);
+ nlohmann::json generation_info(const pipeline::GenerationRequest& request, const sd::VideoConfig& params);
+ nlohmann::json adetailer_generation_info(const sd::ADetailConfig& params, const sd::ImageConfig& inpaint_params);
nlohmann::json upscale_generation_info(uint32_t upscale_factor);
- char* copy_generation_info(const nlohmann::json& info);
- // Consumes images even on failure.
- bool make_image_generation_result(sd_image_t* images, int count, const std::vector<nlohmann::json>& infos, sd_image_generation_result_t** result_out);
+
}
#endif // __SD_GENERATION_INFO_H__
diff --git a/src/model/diffusion/control.hpp b/src/model/diffusion/control.hpp
index 4ad85ec..2fc3dee 100644
--- a/src/model/diffusion/control.hpp
+++ b/src/model/diffusion/control.hpp
@@ -1,6 +1,8 @@
#ifndef __SD_MODEL_DIFFUSION_CONTROL_HPP__
#define __SD_MODEL_DIFFUSION_CONTROL_HPP__
+#include "core/option_types.h"
+
#include "model/common/block.hpp"
#include "stable-diffusion.h"
diff --git a/src/model/integrations/sd.cpp b/src/model/integrations/sd.cpp
index a36ff8e..24a9acf 100644
--- a/src/model/integrations/sd.cpp
+++ b/src/model/integrations/sd.cpp
@@ -1,4 +1,5 @@
#include "conditioning/conditioner.hpp"
+#include "core/option_types.h"
#include "model/diffusion/unet.hpp"
#include "model/vae/auto_encoder_kl.hpp"
#include "model_definition.h"
diff --git a/src/model/vae/minimax_h3_vae.hpp b/src/model/vae/minimax_h3_vae.hpp
index c43c2c7..3010cb8 100644
--- a/src/model/vae/minimax_h3_vae.hpp
+++ b/src/model/vae/minimax_h3_vae.hpp
@@ -1,6 +1,8 @@
#ifndef __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__
#define __SD_MODEL_VAE_MINIMAX_H3_VAE_HPP__
+#include "config.h"
+
#include <algorithm>
#include <array>
#include <cmath>
@@ -719,11 +721,11 @@ namespace MiniMaxH3VAE {
tensor.shape()[3]});
}
- sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const override {
+ sd::TilingConfig resolve_tiling_params(sd::TilingConfig params) const override {
return config.sol ? params : resolve_original_tiling_params(params);
}
- static sd_tiling_params_t resolve_original_tiling_params(sd_tiling_params_t params) {
+ static sd::TilingConfig resolve_original_tiling_params(sd::TilingConfig params) {
if (!params.enabled) {
params.target_overlap = 0.25f;
}
@@ -774,7 +776,7 @@ namespace MiniMaxH3VAE {
sd::Tensor<float> encode(int n_threads,
const sd::Tensor<float>& x,
- sd_tiling_params_t tiling_params,
+ sd::TilingConfig tiling_params,
bool circular_x = false,
bool circular_y = false) override {
auto input = ensure_video_shape(x);
@@ -860,7 +862,7 @@ namespace MiniMaxH3VAE {
sd::Tensor<float> decode(int n_threads,
const sd::Tensor<float>& x,
- sd_tiling_params_t tiling_params,
+ sd::TilingConfig tiling_params,
bool decode_video = false,
bool circular_x = false,
bool circular_y = false,
diff --git a/src/model/vae/vae.hpp b/src/model/vae/vae.hpp
index 3c66965..f0a51c6 100644
--- a/src/model/vae/vae.hpp
+++ b/src/model/vae/vae.hpp
@@ -1,6 +1,8 @@
#ifndef __SD_MODEL_VAE_VAE_HPP__
#define __SD_MODEL_VAE_VAE_HPP__
+#include "config.h"
+
#include <cmath>
#include <limits>
@@ -82,7 +84,7 @@ protected:
sd::Tensor<float> compute_with_temporal_tiling(const int n_threads,
const sd::Tensor<float>& input,
VAETemporalDirection direction,
- const sd_tiling_params_t& tiling_params) {
+ const sd::TilingConfig& tiling_params) {
if (!tiling_params.temporal_tiling || input.dim() != 5 || input.shape()[2] <= 1) {
return _compute(n_threads, input, direction == VAETemporalDirection::DECODE);
}
@@ -126,7 +128,7 @@ protected:
bool circular_x,
bool circular_y,
bool decode_graph,
- const sd_tiling_params_t& tiling_params,
+ const sd::TilingConfig& tiling_params,
const char* error_message,
bool silent = false) {
auto on_processing = [&](const sd::Tensor<float>& input_tile) {
@@ -201,14 +203,14 @@ public:
return supports_temporal_tiling(VAETemporalDirection::DECODE);
}
- virtual sd_tiling_params_t resolve_tiling_params(sd_tiling_params_t params) const {
+ virtual sd::TilingConfig resolve_tiling_params(sd::TilingConfig params) const {
return params;
}
bool get_tile_sizes(int& tile_size_w,
int& tile_size_h,
float& tile_overlap,
- const sd_tiling_params_t& params,
+ const sd::TilingConfig& params,
int64_t latent_w,
int64_t latent_h) {
const auto tiling = resolve_tiling_params(params);
@@ -247,7 +249,7 @@ public:
virtual sd::Tensor<float> encode(int n_threads,
const sd::Tensor<float>& x,
- sd_tiling_params_t tiling_params,
+ sd::TilingConfig tiling_params,
bool circular_x = false,
bool circular_y = false) {
return encode(n_threads, x, tiling_params, circular_x, circular_y, true);
@@ -255,7 +257,7 @@ public:
sd::Tensor<float> encode(int n_threads,
const sd::Tensor<float>& x,
- sd_tiling_params_t tiling_params,
+ sd::TilingConfig tiling_params,
bool circular_x,
bool circular_y,
bool auto_end) {
@@ -313,7 +315,7 @@ public:
virtual sd::Tensor<float> decode(int n_threads,
const sd::Tensor<float>& x,
- sd_tiling_params_t tiling_params,
+ sd::TilingConfig tiling_params,
bool decode_video = false,
bool circular_x = false,
bool circular_y = false,
@@ -323,7 +325,7 @@ public:
sd::Tensor<float> decode(int n_threads,
const sd::Tensor<float>& x,
- sd_tiling_params_t tiling_params,
+ sd::TilingConfig tiling_params,
bool decode_video,
bool circular_x,
bool circular_y,
diff --git a/src/model/vae/vae_tiling.hpp b/src/model/vae/vae_tiling.hpp
index c88ff9f..a343084 100644
--- a/src/model/vae/vae_tiling.hpp
+++ b/src/model/vae/vae_tiling.hpp
@@ -1,6 +1,8 @@
#ifndef __SD_MODEL_VAE_VAE_TILING_HPP__
#define __SD_MODEL_VAE_VAE_TILING_HPP__
+#include "config.h"
+
#include <algorithm>
#include <cstdint>
#include <utility>
@@ -36,7 +38,7 @@ struct VAETemporalTilePlan {
std::vector<VAETemporalTile> tiles;
};
-inline VAETemporalTilingConfig resolve_vae_temporal_tiling_config(const sd_tiling_params_t& params,
+inline VAETemporalTilingConfig resolve_vae_temporal_tiling_config(const sd::TilingConfig& params,
int default_tile_frames,
int default_overlap) {
VAETemporalTilingConfig config;
diff --git a/src/model_definition.h b/src/model_definition.h
index 2b49994..1a0f2df 100644
--- a/src/model_definition.h
+++ b/src/model_definition.h
@@ -1,6 +1,8 @@
#ifndef __SD_MODEL_DEFINITION_H__
#define __SD_MODEL_DEFINITION_H__
+#include "core/option_types.h"
+
#include <limits>
#include "model.h"
#include "stable-diffusion.h"
diff --git a/src/pipeline/diffusion_engine.cpp b/src/pipeline/diffusion_engine.cpp
index 8c33376..c2ae6b8 100644
--- a/src/pipeline/diffusion_engine.cpp
+++ b/src/pipeline/diffusion_engine.cpp
@@ -1,10 +1,13 @@
#include "diffusion_engine.h"
+#include "api/tensor.h"
+#include "core/option_types.h"
#include <algorithm>
#include <cinttypes>
#include <cmath>
#include <cstdlib>
#include <list>
+#include <memory>
#include <mutex>
#include <set>
#include <tuple>
@@ -23,6 +26,7 @@
#include "model.h"
#include "model_definition.h"
+#include "config.h"
#include "core/rng.hpp"
#include "core/rng_mt19937.hpp"
#include "core/rng_philox.hpp"
@@ -890,7 +894,7 @@ bool StableDiffusionGGML::set_sage_attention_enabled(bool enabled) {
return true;
}
-bool StableDiffusionGGML::init(const sd_ctx_params_t* sd_ctx_params) {
+bool StableDiffusionGGML::init(const sd::ContextConfig* sd_ctx_params) {
#ifdef SD_USE_UPSTREAM_GGML
LOG_WARN(
"Using upstream GGML: INT8 tensorwise/convrot is disabled and FP8 weights are "
@@ -1890,12 +1894,12 @@ SDCondition StableDiffusionGGML::get_learned_condition(const ConditionerParams&
return {};
}
if (!sd::resolve_prompt_images(input, [&](const std::string& source) -> sd::Tensor<float> {
- sd_image_t image{};
- if (!prompt_image_callback(source.c_str(), prompt_image_callback_data, &image) ||
- image.data == nullptr || image.width == 0 || image.height == 0 || image.channel < 1 || image.channel > 4) {
+ const sd_tensor_t* tensor = nullptr;
+ if (!prompt_image_callback(source.c_str(), prompt_image_callback_data, &tensor) || !tensor) {
LOG_ERROR("Failed to resolve prompt image: %s", source.c_str());
return {};
}
+ auto image = sd::api::image_view(*tensor);
auto pixels = sd_image_to_tensor(image);
if (image.channel <= 2) {
auto gray = sd::ops::slice(pixels, 2, 0, 1);
@@ -1918,8 +1922,8 @@ void StableDiffusionGGML::reset_generation_extensions() {
}
}
-void StableDiffusionGGML::prepare_generation_extensions(const sd_pm_params_t& pm_params,
- const sd_pulid_params_t& pulid_params,
+void StableDiffusionGGML::prepare_generation_extensions(const sd::PhotoMakerConfig& pm_params,
+ const sd::PuLIDConfig& pulid_params,
ConditionerParams& condition_params,
int total_steps) {
reset_generation_extensions();
@@ -1961,7 +1965,7 @@ sd::Tensor<float> StableDiffusionGGML::get_clip_vision_output(const sd::Tensor<f
}
// Returns 50 Hz wav2vec2 states in sd::Tensor layout: [dim, frames, layers].
-sd::Tensor<float> StableDiffusionGGML::get_audio_embedding(const sd_audio_t& audio) {
+sd::Tensor<float> StableDiffusionGGML::get_audio_embedding(const sd::Audio& audio) {
if (audio_encoder == nullptr) {
LOG_ERROR("audio encoder model is not loaded");
return {};
@@ -1983,7 +1987,7 @@ sd::Tensor<float> StableDiffusionGGML::get_audio_embedding(const sd_audio_t& aud
return audio_encoder->compute(n_threads, mono);
}
-void StableDiffusionGGML::compute_ip_adapter_tokens(const sd_image_t& image, float strength) {
+void StableDiffusionGGML::compute_ip_adapter_tokens(const sd::Image& image, float strength) {
ip_adapter_tokens = {};
ip_adapter_uncond_tokens = {};
ip_adapter_strength = strength;
@@ -2083,7 +2087,7 @@ void StableDiffusionGGML::preview_image(int step,
const sd::Tensor<float>& latents,
enum SDVersion version,
preview_t preview_mode,
- std::function<void(int, int, sd_image_t*, bool, void*)> step_callback,
+ std::function<void(int, int, const sd_tensor_t* const*, bool, void*)> step_callback,
void* step_callback_data,
bool is_noisy) {
bool is_video = preview_latent_tensor_is_video(latents);
@@ -2172,17 +2176,14 @@ void StableDiffusionGGML::preview_image(int step,
uint32_t img_height = static_cast<uint32_t>(_latents.shape()[1]) * patch_sz;
uint32_t img_channels = latent_alpha_proj != nullptr ? 4 : 3;
- uint8_t* data = (uint8_t*)malloc(frames * img_width * img_height * img_channels * sizeof(uint8_t));
+ std::unique_ptr<uint8_t, decltype(&free)> data(static_cast<uint8_t*>(malloc(size_t(frames) * img_width * img_height * img_channels)), free);
GGML_ASSERT(data != nullptr);
- preview_latent_video(data, _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias);
- sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t));
- GGML_ASSERT(images != nullptr);
+ preview_latent_video(data.get(), _latents, latent_rgb_proj, latent_rgb_bias, patch_sz, latent_alpha_proj, latent_alpha_bias);
+ std::vector<sd::Image> images(frames);
for (uint32_t i = 0; i < frames; i++) {
- images[i] = {img_width, img_height, img_channels, data + i * img_width * img_height * img_channels};
+ images[i] = {img_width, img_height, img_channels, data.get() + size_t(i) * img_width * img_height * img_channels};
}
- step_callback(step, frames, images, is_noisy, step_callback_data);
- free(data);
- free(images);
+ sd::api::preview_images(step_callback, step, frames, images.data(), is_noisy, step_callback_data);
return;
}
@@ -2201,19 +2202,20 @@ void StableDiffusionGGML::preview_image(int step,
return;
}
- is_video = preview_latent_tensor_is_video(decoded);
- uint32_t frames = is_video ? static_cast<uint32_t>(decoded.shape()[2]) : 1;
- sd_image_t* images = (sd_image_t*)malloc(frames * sizeof(sd_image_t));
+ is_video = preview_latent_tensor_is_video(decoded);
+ uint32_t frames = is_video ? static_cast<uint32_t>(decoded.shape()[2]) : 1;
+ auto release_images = [frames](sd::Image* images) {
+ for (uint32_t i = 0; i < frames; ++i)
+ free(images[i].data);
+ free(images);
+ };
+ std::unique_ptr<sd::Image, decltype(release_images)> images(static_cast<sd::Image*>(calloc(frames, sizeof(sd::Image))), release_images);
GGML_ASSERT(images != nullptr);
for (uint32_t i = 0; i < frames; ++i) {
- images[i] = tensor_to_sd_image(decoded, static_cast<int>(i));
+ images.get()[i] = tensor_to_sd_image(decoded, static_cast<int>(i));
}
- step_callback(step, frames, images, is_noisy, step_callback_data);
- for (uint32_t i = 0; i < frames; ++i) {
- free(images[i].data);
- }
- free(images);
+ sd::api::preview_images(step_callback, step, frames, images.get(), is_noisy, step_callback_data);
return;
}
@@ -2352,7 +2354,7 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
const SDCondition& img_uncond,
const sd::Tensor<float>& control_image,
float control_strength,
- const sd_guidance_params_t& guidance,
+ const sd::GuidanceConfig& guidance,
float eta,
int shifted_timestep,
sample_method_t method,
@@ -2366,7 +2368,7 @@ sd::Tensor<float> StableDiffusionGGML::sample(const std::shared_ptr<DiffusionMod
float vace_strength,
int audio_length,
float frame_rate,
- const sd_cache_params_t* cache_params,
+ const sd::CacheConfig* cache_params,
bool preview_final_step,
const sd::Tensor<float>& video_positions) {
struct RunnerEndOnExit {
diff --git a/src/pipeline/diffusion_engine.h b/src/pipeline/diffusion_engine.h
index d2a05e0..ae05dc9 100644
--- a/src/pipeline/diffusion_engine.h
+++ b/src/pipeline/diffusion_engine.h
@@ -1,6 +1,10 @@
#ifndef __SD_PIPELINE_DIFFUSION_ENGINE_H__
#define __SD_PIPELINE_DIFFUSION_ENGINE_H__
+#include "core/media.h"
+
+#include "core/option_types.h"
+
#include <atomic>
#include <cmath>
#include <functional>
@@ -14,6 +18,7 @@
#include <unordered_map>
#include <vector>
+#include "config.h"
#include "core/ggml_extend_backend.h"
#include "core/ggml_graph_cut.h"
#include "core/tensor.hpp"
@@ -99,8 +104,8 @@ public:
int animatediff_num_frames = 0;
std::string taesd_path;
- sd_tiling_params_t vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr};
- bool enable_mmap = false;
+ sd::TilingConfig vae_tiling_params = {false, false, 0, 0, 0.5f, 0, 0, nullptr};
+ bool enable_mmap = false;
sd::ggml_graph_cut::MaxVramAssignment max_vram_assignment;
bool disable_prefetch = false;
bool disable_segmented_compute = false;
@@ -124,7 +129,7 @@ public:
using RunnerGroups = std::set<RunnerGroup>;
struct ModelConfig {
- sd_ctx_params_t params{};
+ sd::ContextConfig params{};
std::list<std::string> strings;
std::vector<sd_embedding_t> embeddings;
std::map<std::string, std::string> models;
@@ -134,22 +139,22 @@ public:
bool photomaker_source_available = false;
bool animatediff_loaded = false;
- explicit ModelConfig(const sd_ctx_params_t& initial)
+ explicit ModelConfig(const sd::ContextConfig& initial)
: params(initial) {
- for (auto member : {&sd_ctx_params_t::model_path, &sd_ctx_params_t::clip_l_path,
- &sd_ctx_params_t::clip_g_path, &sd_ctx_params_t::clip_vision_path,
- &sd_ctx_params_t::t5xxl_path, &sd_ctx_params_t::llm_path,
- &sd_ctx_params_t::llm_vision_path, &sd_ctx_params_t::diffusion_model_path,
- &sd_ctx_params_t::tokenizer, &sd_ctx_params_t::flash_attn_mode, &sd_ctx_params_t::conv_direct_mode,
- &sd_ctx_params_t::high_noise_diffusion_model_path, &sd_ctx_params_t::uncond_diffusion_model_path,
- &sd_ctx_params_t::embeddings_connectors_path, &sd_ctx_params_t::vae_path,
- &sd_ctx_params_t::audio_vae_path, &sd_ctx_params_t::taesd_path,
- &sd_ctx_params_t::control_net_path, &sd_ctx_params_t::ip_adapter_path,
- &sd_ctx_params_t::motion_module_path, &sd_ctx_params_t::photo_maker_path,
- &sd_ctx_params_t::pulid_weights_path, &sd_ctx_params_t::tensor_type_rules,
- &sd_ctx_params_t::max_vram, &sd_ctx_params_t::backend,
- &sd_ctx_params_t::params_backend, &sd_ctx_params_t::split_mode,
- &sd_ctx_params_t::rpc_servers, &sd_ctx_params_t::model_args}) {
+ for (auto member : {&sd::ContextConfig::model_path, &sd::ContextConfig::clip_l_path,
+ &sd::ContextConfig::clip_g_path, &sd::ContextConfig::clip_vision_path,
+ &sd::ContextConfig::t5xxl_path, &sd::ContextConfig::llm_path,
+ &sd::ContextConfig::llm_vision_path, &sd::ContextConfig::diffusion_model_path,
+ &sd::ContextConfig::tokenizer, &sd::ContextConfig::flash_attn_mode, &sd::ContextConfig::conv_direct_mode,
+ &sd::ContextConfig::high_noise_diffusion_model_path, &sd::ContextConfig::uncond_diffusion_model_path,
+ &sd::ContextConfig::embeddings_connectors_path, &sd::ContextConfig::vae_path,
+ &sd::ContextConfig::audio_vae_path, &sd::ContextConfig::audio_encoder_path, &sd::ContextConfig::taesd_path,
+ &sd::ContextConfig::control_net_path, &sd::ContextConfig::ip_adapter_path,
+ &sd::ContextConfig::motion_module_path, &sd::ContextConfig::photo_maker_path,
+ &sd::ContextConfig::pulid_weights_path, &sd::ContextConfig::tensor_type_rules,
+ &sd::ContextConfig::max_vram, &sd::ContextConfig::backend,
+ &sd::ContextConfig::params_backend, &sd::ContextConfig::split_mode,
+ &sd::ContextConfig::rpc_servers, &sd::ContextConfig::model_args}) {
strings.emplace_back(SAFE_STR(initial.*member));
params.*member = strings.back().c_str();
}
@@ -338,7 +343,7 @@ public:
bool init_model_loader(ModelLoader& model_loader, ModelConfig& configuration);
- bool init(const sd_ctx_params_t* sd_ctx_params);
+ bool init(const sd::ContextConfig* sd_ctx_params);
bool set_sage_attention_enabled(bool enabled);
bool uses_tae() const;
@@ -394,8 +399,8 @@ public:
void reset_generation_extensions();
- void prepare_generation_extensions(const sd_pm_params_t& pm_params,
- const sd_pulid_params_t& pulid_params,
+ void prepare_generation_extensions(const sd::PhotoMakerConfig& pm_params,
+ const sd::PuLIDConfig& pulid_params,
ConditionerParams& condition_params,
int total_steps);
@@ -404,9 +409,9 @@ public:
int clip_skip = -1,
bool zero_out_masked = false);
- sd::Tensor<float> get_audio_embedding(const sd_audio_t& audio);
+ sd::Tensor<float> get_audio_embedding(const sd::Audio& audio);
- void compute_ip_adapter_tokens(const sd_image_t& image, float strength);
+ void compute_ip_adapter_tokens(const sd::Image& image, float strength);
std::vector<float> process_timesteps(const std::vector<float>& timesteps,
const sd::Tensor<float>& init_latent,
@@ -421,7 +426,7 @@ public:
const sd::Tensor<float>& latents,
enum SDVersion version,
preview_t preview_mode,
- std::function<void(int, int, sd_image_t*, bool, void*)> step_callback,
+ std::function<void(int, int, const sd_tensor_t* const*, bool, void*)> step_callback,
void* step_callback_data,
bool is_noisy);
@@ -466,7 +471,7 @@ public:
const SDCondition& img_uncond,
const sd::Tensor<float>& control_image,
float control_strength,
- const sd_guidance_params_t& guidance,
+ const sd::GuidanceConfig& guidance,
float eta,
int shifted_timestep,
sample_method_t method,
@@ -480,7 +485,7 @@ public:
float vace_strength,
int audio_length,
float frame_rate,
- const sd_cache_params_t* cache_params,
+ const sd::CacheConfig* cache_params,
bool preview_final_step,
const sd::Tensor<float>& video_positions = {});
diff --git a/src/pipeline/generation.h b/src/pipeline/generation.h
index 0bc3913..08511f2 100644
--- a/src/pipeline/generation.h
+++ b/src/pipeline/generation.h
@@ -1,12 +1,22 @@
#ifndef __SD_PIPELINE_GENERATION_H__
#define __SD_PIPELINE_GENERATION_H__
+#include "core/media.h"
+
#include "conditioning/conditioner.hpp"
+#include "config.h"
#include "json.hpp"
#include "stable-diffusion.h"
class StableDiffusionGGML;
+namespace sd {
+ struct ImageGenerationResult;
+ struct VideoGenerationResult;
+ sd_status_t generate_image(sd_ctx_t* ctx, const ImageConfig& config, ImageGenerationResult** result_out, const sd_opt_t* source = nullptr) noexcept;
+ sd_status_t generate_video(sd_ctx_t* ctx, const VideoConfig& config, VideoGenerationResult** result_out, const sd_opt_t* source = nullptr) noexcept;
+}
+
static inline bool sd_version_supports_animatediff(SDVersion version) {
return version == VERSION_SD1 || version == VERSION_SD1_INPAINT || version == VERSION_SD1_PIX2PIX;
}
@@ -53,16 +63,16 @@ namespace sd::pipeline {
// Callers hold ExecutionScope; AnimateDiff reuses the image path within the same scope.
bool generate_image(StableDiffusionGGML* sd,
- const sd_img_gen_params_t* sd_img_gen_params,
- sd_image_t** images_out,
+ const sd::ImageConfig* sd_img_gen_params,
+ sd::Image** images_out,
int* num_images_out,
std::vector<nlohmann::json>* infos_out = nullptr);
bool generate_video(StableDiffusionGGML* sd,
- const sd_vid_gen_params_t* sd_vid_gen_params,
- sd_image_t** frames_out,
+ const sd::VideoConfig* sd_vid_gen_params,
+ sd::Image** frames_out,
int* num_frames_out,
- sd_audio_t** audio_out,
+ sd::Audio** audio_out,
int* fps_out,
nlohmann::json* info_out = nullptr);
diff --git a/src/pipeline/image.cpp b/src/pipeline/image.cpp
index 5e7a349..f2cdcea 100644
--- a/src/pipeline/image.cpp
+++ b/src/pipeline/image.cpp
@@ -1,3 +1,5 @@
+#include "api/result.h"
+#include "core/option_types.h"
#include "generation.h"
#include "generation_info.h"
@@ -22,7 +24,7 @@ namespace sd::pipeline {
};
static CircularAxesState configure_image_vae_axes(StableDiffusionGGML* sd,
- const sd_img_gen_params_t* sd_img_gen_params,
+ const sd::ImageConfig* sd_img_gen_params,
const GenerationRequest& request) {
CircularAxesState original_axes = {sd->circular_x, sd->circular_y};
@@ -77,7 +79,7 @@ namespace sd::pipeline {
public:
ImageVaeAxesGuard(StableDiffusionGGML* sd,
- const sd_img_gen_params_t* sd_img_gen_params,
+ const sd::ImageConfig* sd_img_gen_params,
const GenerationRequest& request)
: sd(sd),
original_axes(configure_image_vae_axes(sd, sd_img_gen_params, request)) {}
@@ -184,7 +186,7 @@ namespace sd::pipeline {
}
static std::optional<ImageGenerationLatents> prepare_image_generation_latents(StableDiffusionGGML* sd,
- const sd_img_gen_params_t* sd_img_gen_params,
+ const sd::ImageConfig* sd_img_gen_params,
GenerationRequest* request,
SamplePlan* plan,
const RefImageParams& ref_image_params) {
@@ -541,7 +543,7 @@ namespace sd::pipeline {
}
static std::optional<ImageGenerationEmbeds> prepare_image_generation_embeds(StableDiffusionGGML* sd,
- const sd_img_gen_params_t* sd_img_gen_params,
+ const sd::ImageConfig* sd_img_gen_params,
GenerationRequest* request,
SamplePlan* plan,
ImageGenerationLatents* latents,
@@ -669,10 +671,10 @@ namespace sd::pipeline {
return embeds;
}
- static sd_image_t* decode_image_outputs(StableDiffusionGGML* sd,
- const GenerationRequest& request,
- const std::vector<sd::Tensor<float>>& final_latents,
- int* num_images_out) {
+ static sd::Image* decode_image_outputs(StableDiffusionGGML* sd,
+ const GenerationRequest& request,
+ const std::vector<sd::Tensor<float>>& final_latents,
+ int* num_images_out) {
if (final_latents.empty()) {
LOG_ERROR("no latent images to decode");
return nullptr;
@@ -760,8 +762,8 @@ namespace sd::pipeline {
return nullptr;
}
- int image_count = static_cast<int>(decoded_images.size());
- sd_image_t* result_images = (sd_image_t*)calloc(image_count, sizeof(sd_image_t));
+ int image_count = static_cast<int>(decoded_images.size());
+ sd::Image* result_images = (sd::Image*)calloc(image_count, sizeof(sd::Image));
if (result_images == nullptr) {
return nullptr;
}
@@ -915,8 +917,8 @@ namespace sd::pipeline {
}
bool generate_image(StableDiffusionGGML* sd,
- const sd_img_gen_params_t* sd_img_gen_params,
- sd_image_t** images_out,
+ const sd::ImageConfig* sd_img_gen_params,
+ sd::Image** images_out,
int* num_images_out,
std::vector<nlohmann::json>* infos_out) {
if (infos_out)
@@ -944,7 +946,7 @@ namespace sd::pipeline {
GenerationRequest request(sd, sd_img_gen_params);
const auto info = sd::generation_info(request, *sd_img_gen_params);
sd::ImagePreprocessor preprocessing(sd_img_gen_params->image_preprocess.rules);
- sd_img_gen_params_t processed_params = *sd_img_gen_params;
+ sd::ImageConfig processed_params = *sd_img_gen_params;
if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
return false;
sd_img_gen_params = &processed_params;
@@ -953,7 +955,6 @@ namespace sd::pipeline {
sd->rng->manual_seed(request.seed);
sd->sampler_rng->manual_seed(request.seed);
- sd->set_flow_shift(sd_img_gen_params->sample_params.flow_shift);
if (!sd->apply_loras(sd_img_gen_params->loras, sd_img_gen_params->lora_count))
return false;
sd->apply_circular_axes(sd_img_gen_params->circular_x, sd_img_gen_params->circular_y);
@@ -1084,7 +1085,6 @@ namespace sd::pipeline {
make_hires_sigma_schedule(sd,
request.hires,
request.sample_params,
- plan.sample_method,
plan.sample_steps,
sd->get_image_seq_len(request.hires.target_height, request.hires.target_width),
&hires_scheduler_steps);
@@ -1217,7 +1217,7 @@ namespace sd::pipeline {
(*infos_out)[i]["seed"] = std::to_string(request.seed_for_batch(i / images_per_latent));
}
} catch (...) {
- free_sd_images(result, num_images);
+ sd::free_images(result, num_images);
throw;
}
}
@@ -1231,7 +1231,7 @@ namespace sd::pipeline {
if (images_out != nullptr) {
*images_out = result;
} else {
- free_sd_images(result, num_images);
+ sd::free_images(result, num_images);
}
return true;
}
diff --git a/src/pipeline/model_builders.cpp b/src/pipeline/model_builders.cpp
index afc0ce8..389b199 100644
--- a/src/pipeline/model_builders.cpp
+++ b/src/pipeline/model_builders.cpp
@@ -1,4 +1,5 @@
#include "model_builders.h"
+#include "core/option_types.h"
#include <cmath>
#include <cstring>
diff --git a/src/pipeline/model_builders.h b/src/pipeline/model_builders.h
index bb51ccc..1f56e4a 100644
--- a/src/pipeline/model_builders.h
+++ b/src/pipeline/model_builders.h
@@ -4,6 +4,7 @@
#include <memory>
#include <vector>
+#include "config.h"
#include "core/ggml_extend_backend.h"
#include "model.h"
#include "stable-diffusion.h"
@@ -28,7 +29,7 @@ namespace IPAdapter {
namespace sd::model_builders {
struct Context {
- const sd_ctx_params_t& params;
+ const sd::ContextConfig& params;
SDVersion version;
const String2TensorStorage& tensor_storage_map;
SDBackendManager& backends;
diff --git a/src/pipeline/relay_vsr.cpp b/src/pipeline/relay_vsr.cpp
index c8b0d72..20d70ea 100644
--- a/src/pipeline/relay_vsr.cpp
+++ b/src/pipeline/relay_vsr.cpp
@@ -1,3 +1,4 @@
+#include "api/result.h"
#include "pipeline/video_upscale.h"
#include <climits>
@@ -11,7 +12,7 @@
namespace sd {
namespace pipeline {
- static sd::Tensor<float> prepare_vsr_frame(const sd_image_t& frame, int width, int height) {
+ static sd::Tensor<float> prepare_vsr_frame(const sd::Image& frame, int width, int height) {
auto result = sd_image_to_tensor(frame, -1, -1, false);
result /= 127.5f;
result -= 1.f;
@@ -103,7 +104,7 @@ namespace sd {
flash_->runner_end();
}
- bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) override {
+ bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) override {
const auto* frames = params.frames;
const int frame_count = params.frame_count, fps = params.fps, window_size = params.window_size;
const int64_t seed = params.seed;
diff --git a/src/pipeline/request.cpp b/src/pipeline/request.cpp
index 739b8e4..bbf3ab7 100644
--- a/src/pipeline/request.cpp
+++ b/src/pipeline/request.cpp
@@ -1,5 +1,7 @@
#include "request.h"
#include "conditioning/conditioner.hpp"
+#include "config.h"
+#include "core/option_types.h"
#include <algorithm>
#include <cmath>
@@ -93,9 +95,9 @@ namespace sd::pipeline {
return scheduler;
}
- float resolve_eta(StableDiffusionGGML* sd,
- float eta,
- enum sample_method_t sample_method) {
+ static float resolve_eta(StableDiffusionGGML* sd,
+ float eta,
+ enum sample_method_t sample_method) {
if (eta == INFINITY) {
if (sd->version == VERSION_HIDREAM_O1) {
return 8.f;
@@ -120,7 +122,7 @@ namespace sd::pipeline {
return eta;
}
- GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params) {
+ GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd::ImageConfig* sd_img_gen_params) {
prompt = SAFE_STR(sd_img_gen_params->prompt);
negative_prompt = SAFE_STR(sd_img_gen_params->negative_prompt);
width = sd_img_gen_params->width;
@@ -136,7 +138,6 @@ namespace sd::pipeline {
strength = sd_img_gen_params->strength;
control_strength = sd_img_gen_params->control_strength;
control_type = sd_img_gen_params->control_type;
- eta = sd_img_gen_params->sample_params.eta;
has_ref_images = sd_img_gen_params->ref_images_count > 0;
guidance = sd_img_gen_params->sample_params.guidance;
pm_params = sd_img_gen_params->pm_params;
@@ -156,7 +157,7 @@ namespace sd::pipeline {
models.erase("pulid_weights");
}
- GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params) {
+ GenerationRequest::GenerationRequest(StableDiffusionGGML* sd, const sd::VideoConfig* sd_vid_gen_params) {
prompt = SAFE_STR(sd_vid_gen_params->prompt);
negative_prompt = SAFE_STR(sd_vid_gen_params->negative_prompt);
width = sd_vid_gen_params->width;
@@ -293,7 +294,7 @@ namespace sd::pipeline {
}
void GenerationRequest::resolve_guidance(StableDiffusionGGML* sd,
- sd_guidance_params_t* guidance,
+ sd::GuidanceConfig* guidance,
bool* use_uncond,
bool* use_img_uncond,
bool has_ref_images,
@@ -388,16 +389,21 @@ namespace sd::pipeline {
sample_params.sample_method = resolve_sample_method(sd, sample_params.sample_method);
sample_params.scheduler = resolve_scheduler(sd, sample_params.scheduler, sample_params.sample_method);
sample_params.eta = resolve_eta(sd, sample_params.eta, sample_params.sample_method);
+ sd->set_flow_shift(sample_params.flow_shift);
+ if (sd->is_flow_denoiser())
+ sample_params.flow_shift = sd->active_flow_shift;
if (frames > 0) {
- high_noise_sample_params.guidance = high_noise_guidance;
- high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method);
- high_noise_sample_params.scheduler = resolve_scheduler(sd, high_noise_sample_params.scheduler, high_noise_sample_params.sample_method);
- high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method);
+ high_noise_sample_params.guidance = high_noise_guidance;
+ high_noise_sample_params.sample_method = resolve_sample_method(sd, high_noise_sample_params.sample_method);
+ high_noise_sample_params.scheduler = sample_params.scheduler;
+ high_noise_sample_params.flow_shift = sample_params.flow_shift;
+ high_noise_sample_params.shifted_timestep = sample_params.shifted_timestep;
+ high_noise_sample_params.eta = resolve_eta(sd, high_noise_sample_params.eta, high_noise_sample_params.sample_method);
}
}
SamplePlan::SamplePlan(StableDiffusionGGML* sd,
- const sd_img_gen_params_t* sd_img_gen_params,
+ const sd::ImageConfig* sd_img_gen_params,
const GenerationRequest& request) {
sample_method = request.sample_params.sample_method;
extra_sample_args = request.sample_params.extra_sample_args;
@@ -407,7 +413,7 @@ namespace sd::pipeline {
}
SamplePlan::SamplePlan(StableDiffusionGGML* sd,
- const sd_vid_gen_params_t* sd_vid_gen_params,
+ const sd::VideoConfig* sd_vid_gen_params,
const GenerationRequest& request) {
sample_method = request.sample_params.sample_method;
extra_sample_args = request.sample_params.extra_sample_args;
@@ -425,9 +431,8 @@ namespace sd::pipeline {
void SamplePlan::resolve(StableDiffusionGGML* sd,
const GenerationRequest* request,
- const sd_sample_params_t* sample_params) {
- sample_method = resolve_sample_method(sd, sample_method);
-
+ const sd::SampleConfig* sample_params) {
+ scheduler = sample_params->scheduler;
total_steps = sample_steps + std::max(0, high_noise_sample_steps);
if (sample_params->custom_sigmas_count > 0) {
@@ -446,9 +451,6 @@ namespace sd::pipeline {
}
}
} else {
- scheduler = resolve_scheduler(sd,
- sample_params->scheduler,
- sample_method);
int sample_seq_len = sd->get_image_seq_len(request->height, request->width);
if (sd_version_is_ltxav(sd->version) && request->frames > 0) {
int latent_frames = ((request->frames - 1) / 8) + 1;
@@ -463,8 +465,6 @@ namespace sd::pipeline {
sample_params->extra_sample_args);
}
- eta = resolve_eta(sd, eta, sample_method);
-
if (high_noise_sample_steps < 0) {
for (size_t i = 0; i < sigmas.size(); ++i) {
if (sigmas[i] < moe_boundary) {
@@ -477,17 +477,13 @@ namespace sd::pipeline {
LOG_INFO("sampling using %s method", sampling_methods_str[sample_method]);
if (high_noise_sample_steps > 0) {
- high_noise_sample_method = resolve_sample_method(sd,
- high_noise_sample_method);
- high_noise_eta = resolve_eta(sd, high_noise_eta, high_noise_sample_method);
LOG_INFO("sampling(high noise) using %s method", sampling_methods_str[high_noise_sample_method]);
}
}
std::vector<float> make_hires_sigma_schedule(StableDiffusionGGML* sd,
- const sd_hires_params_t& hires,
- const sd_sample_params_t& sample_params,
- sample_method_t sample_method,
+ const sd::HiresConfig& hires,
+ const sd::SampleConfig& sample_params,
int default_steps,
int sample_seq_len,
int* scheduler_steps_out) {
@@ -512,9 +508,7 @@ namespace sd::pipeline {
int scheduler_steps = static_cast<int>(effective_steps / hires.denoising_strength);
scheduler_steps = std::max(1, scheduler_steps);
- scheduler_t scheduler = resolve_scheduler(sd,
- sample_params.scheduler,
- sample_method);
+ scheduler_t scheduler = sample_params.scheduler;
std::vector<float> sigmas = sd->denoiser->get_sigmas(scheduler_steps,
sample_seq_len,
scheduler,
diff --git a/src/pipeline/request.h b/src/pipeline/request.h
index b1ab640..1e15db8 100644
--- a/src/pipeline/request.h
+++ b/src/pipeline/request.h
@@ -1,10 +1,13 @@
#ifndef __SD_PIPELINE_REQUEST_H__
#define __SD_PIPELINE_REQUEST_H__
+#include "core/option_types.h"
+
#include <map>
#include <string>
#include <vector>
+#include "config.h"
#include "stable-diffusion.h"
class StableDiffusionGGML;
@@ -19,52 +22,47 @@ namespace sd::pipeline {
int64_t resolve_seed(int64_t seed);
- float resolve_eta(StableDiffusionGGML* sd,
- float eta,
- enum sample_method_t sample_method);
-
struct GenerationRequest {
std::string prompt;
std::string negative_prompt;
std::map<std::string, std::string> models;
- int width = -1;
- int height = -1;
- int clip_skip = -1;
- int vae_scale_factor = -1;
- int diffusion_model_down_factor = -1;
- int64_t seed = -1;
- bool use_uncond = false;
- bool use_img_uncond = false;
- bool use_high_noise_uncond = false;
- bool use_high_noise_img_uncond = false;
- bool has_ref_images = false;
- bool is_layered_image = false;
- bool use_high_noise = false;
- bool use_vace = false;
- const sd_cache_params_t* cache_params = nullptr;
- int batch_count = 1;
- int qwen_image_layers = 3;
- int shifted_timestep = 0;
- float strength = 1.f;
- float control_strength = 0.f;
- sd_control_type_t control_type = SD_CONTROL_TYPE_NONE;
- float eta = 0.f;
- sd_sample_params_t sample_params = {};
- sd_sample_params_t high_noise_sample_params = {};
- sd_guidance_params_t guidance = {};
- sd_guidance_params_t high_noise_guidance = {};
- sd_pm_params_t pm_params = {};
- sd_pulid_params_t pulid_params = {};
- sd_hires_params_t hires = {};
- sd_tiling_params_t vae_tiling_params = {};
- int frames = -1;
- int requested_frames = -1;
- int fps = 16;
- float vace_strength = 1.f;
-
- GenerationRequest(StableDiffusionGGML* sd, const sd_img_gen_params_t* sd_img_gen_params);
-
- GenerationRequest(StableDiffusionGGML* sd, const sd_vid_gen_params_t* sd_vid_gen_params);
+ int width = -1;
+ int height = -1;
+ int clip_skip = -1;
+ int vae_scale_factor = -1;
+ int diffusion_model_down_factor = -1;
+ int64_t seed = -1;
+ bool use_uncond = false;
+ bool use_img_uncond = false;
+ bool use_high_noise_uncond = false;
+ bool use_high_noise_img_uncond = false;
+ bool has_ref_images = false;
+ bool is_layered_image = false;
+ bool use_high_noise = false;
+ bool use_vace = false;
+ const sd::CacheConfig* cache_params = nullptr;
+ int batch_count = 1;
+ int qwen_image_layers = 3;
+ int shifted_timestep = 0;
+ float strength = 1.f;
+ float control_strength = 0.f;
+ sd_control_type_t control_type = SD_CONTROL_TYPE_NONE;
+ sd::SampleConfig sample_params = {};
+ sd::SampleConfig high_noise_sample_params = {};
+ sd::GuidanceConfig guidance = {};
+ sd::GuidanceConfig high_noise_guidance = {};
+ sd::PhotoMakerConfig pm_params = {};
+ sd::PuLIDConfig pulid_params = {};
+ sd::HiresConfig hires = {};
+ sd::TilingConfig vae_tiling_params = {};
+ int frames = -1;
+ int requested_frames = -1;
+ int fps = 16;
+ float vace_strength = 1.f;
+
+ GenerationRequest(StableDiffusionGGML* sd, const sd::ImageConfig* sd_img_gen_params);
+
+ GenerationRequest(StableDiffusionGGML* sd, const sd::VideoConfig* sd_vid_gen_params);
int64_t seed_for_batch(int index) const;
@@ -75,7 +73,7 @@ namespace sd::pipeline {
void resolve_hires();
static void resolve_guidance(StableDiffusionGGML* sd,
- sd_guidance_params_t* guidance,
+ sd::GuidanceConfig* guidance,
bool* use_uncond,
bool* use_img_uncond,
bool has_ref_images,
@@ -99,22 +97,21 @@ namespace sd::pipeline {
std::vector<float> sigmas;
SamplePlan(StableDiffusionGGML* sd,
- const sd_img_gen_params_t* sd_img_gen_params,
+ const sd::ImageConfig* sd_img_gen_params,
const GenerationRequest& request);
SamplePlan(StableDiffusionGGML* sd,
- const sd_vid_gen_params_t* sd_vid_gen_params,
+ const sd::VideoConfig* sd_vid_gen_params,
const GenerationRequest& request);
void resolve(StableDiffusionGGML* sd,
const GenerationRequest* request,
- const sd_sample_params_t* sample_params);
+ const sd::SampleConfig* sample_params);
};
std::vector<float> make_hires_sigma_schedule(StableDiffusionGGML* sd,
- const sd_hires_params_t& hires,
- const sd_sample_params_t& sample_params,
- sample_method_t sample_method,
+ const sd::HiresConfig& hires,
+ const sd::SampleConfig& sample_params,
int default_steps,
int sample_seq_len,
int* scheduler_steps_out);
diff --git a/src/pipeline/seedvr2.cpp b/src/pipeline/seedvr2.cpp
index 5fe2eaa..cb07876 100644
--- a/src/pipeline/seedvr2.cpp
+++ b/src/pipeline/seedvr2.cpp
@@ -1,3 +1,4 @@
+#include "api/result.h"
#include "pipeline/video_upscale.h"
#include <climits>
@@ -54,7 +55,7 @@ namespace sd {
vae_->runner_end();
}
- bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) override {
+ bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) override {
const int chunk = params.temporal_chunk_size;
if (chunk < 1 || chunk > INT_MAX - 3 || (chunk - 1) % 4) {
LOG_ERROR("SeedVR2 temporal chunk size must be 1 or 4n+1 input frames");
diff --git a/src/pipeline/video.cpp b/src/pipeline/video.cpp
index ab64540..529bbb6 100644
--- a/src/pipeline/video.cpp
+++ b/src/pipeline/video.cpp
@@ -1,3 +1,4 @@
+#include "core/option_types.h"
#include "generation.h"
#include "generation_info.h"
@@ -22,8 +23,8 @@
namespace sd::pipeline {
- static sd_audio_t* waveform_to_sd_audio(const StableDiffusionGGML* sd,
- const sd::Tensor<float>& waveform) {
+ static sd::Audio* waveform_to_sd_audio(const StableDiffusionGGML* sd,
+ const sd::Tensor<float>& waveform) {
if (sd == nullptr || waveform.empty()) {
return nullptr;
}
@@ -34,7 +35,7 @@ namespace sd::pipeline {
return nullptr;
}
- sd_audio_t* audio = (sd_audio_t*)malloc(sizeof(sd_audio_t));
+ sd::Audio* audio = (sd::Audio*)malloc(sizeof(sd::Audio));
if (audio == nullptr) {
return nullptr;
}
@@ -374,7 +375,7 @@ namespace sd::pipeline {
return sd::zeros<float>({audio_length, 2, 32, 1});
}
- static sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd_audio_t& audio,
+ static sd::Tensor<float> prepare_minimax_h3_reference_waveform(const sd::Audio& audio,
int target_sample_rate = 32000) {
if (audio.data == nullptr || audio.sample_count == 0 || audio.channels == 0 || audio.sample_rate == 0) {
return {};
@@ -490,7 +491,7 @@ namespace sd::pipeline {
}
static std::optional<ImageGenerationLatents> prepare_video_generation_latents(StableDiffusionGGML* sd,
- const sd_vid_gen_params_t* sd_vid_gen_params,
+ const sd::VideoConfig* sd_vid_gen_params,
GenerationRequest* request,
const RefImageParams& ref_image_params) {
ImageGenerationLatents latents;
@@ -576,7 +577,7 @@ namespace sd::pipeline {
std::vector<int64_t>{width, height, image.shape()[2], image.shape()[3]});
};
- auto encode_reference_audio = [&](const sd_audio_t& audio,
+ auto encode_reference_audio = [&](const sd::Audio& audio,
int32_t* audio_index) -> bool {
if (sd->audio_vae_model == nullptr) {
LOG_ERROR("MiniMax-H3 Ref2VA audio requires --audio-vae with encoder weights");
@@ -1194,7 +1195,7 @@ namespace sd::pipeline {
}
static std::optional<ImageGenerationEmbeds> prepare_video_generation_embeds(StableDiffusionGGML* sd,
- const sd_vid_gen_params_t* sd_vid_gen_params,
+ const sd::VideoConfig* sd_vid_gen_params,
const GenerationRequest& request,
const ImageGenerationLatents& latents,
const RefImageParams& ref_image_params) {
@@ -1264,10 +1265,10 @@ namespace sd::pipeline {
return embeds;
}
- static sd_image_t* decode_video_outputs(StableDiffusionGGML* sd,
- const GenerationRequest& request,
- const sd::Tensor<float>& final_latent,
- int* num_frames_out) {
+ static sd::Image* decode_video_outputs(StableDiffusionGGML* sd,
+ const GenerationRequest& request,
+ const sd::Tensor<float>& final_latent,
+ int* num_frames_out) {
if (final_latent.empty()) {
LOG_ERROR("no latent video to decode");
return nullptr;
@@ -1305,7 +1306,7 @@ namespace sd::pipeline {
vid = sd::ops::slice(vid, 2, 0, request.frames);
}
- sd_image_t* result_images = (sd_image_t*)calloc(vid.shape()[2], sizeof(sd_image_t));
+ sd::Image* result_images = (sd::Image*)calloc(vid.shape()[2], sizeof(sd::Image));
if (result_images == nullptr) {
return nullptr;
}
@@ -1661,7 +1662,7 @@ namespace sd::pipeline {
}
static bool apply_ltxv_refine_image_conditioning(StableDiffusionGGML* sd,
- const sd_vid_gen_params_t* sd_vid_gen_params,
+ const sd::VideoConfig* sd_vid_gen_params,
const GenerationRequest& request,
const ImageGenerationLatents& latents,
sd::Tensor<float>* latent,
@@ -1765,8 +1766,8 @@ namespace sd::pipeline {
}
static bool generate_animatediff_video(StableDiffusionGGML* sd,
- const sd_vid_gen_params_t* sd_vid_gen_params,
- sd_image_t** frames_out,
+ const sd::VideoConfig* sd_vid_gen_params,
+ sd::Image** frames_out,
int* num_frames_out) {
int n_frames = sd_vid_gen_params->video_frames;
if (n_frames < 1) {
@@ -1778,8 +1779,8 @@ namespace sd::pipeline {
n_frames = 32;
}
- sd_img_gen_params_t img_gen_params;
- sd_img_gen_params_init(&img_gen_params);
+ sd::ImageConfig img_gen_params;
+
img_gen_params.loras = sd_vid_gen_params->loras;
img_gen_params.lora_count = sd_vid_gen_params->lora_count;
img_gen_params.prompt = sd_vid_gen_params->prompt;
@@ -1815,10 +1816,10 @@ namespace sd::pipeline {
}
bool generate_video(StableDiffusionGGML* sd,
- const sd_vid_gen_params_t* sd_vid_gen_params,
- sd_image_t** frames_out,
+ const sd::VideoConfig* sd_vid_gen_params,
+ sd::Image** frames_out,
int* num_frames_out,
- sd_audio_t** audio_out,
+ sd::Audio** audio_out,
int* fps_out,
nlohmann::json* info_out) {
if (info_out)
@@ -1847,7 +1848,7 @@ namespace sd::pipeline {
sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
sd->apply_circular_axes(sd_vid_gen_params->circular_x, sd_vid_gen_params->circular_y);
sd::ImagePreprocessor preprocessing(sd_vid_gen_params->image_preprocess.rules);
- sd_vid_gen_params_t processed_params = *sd_vid_gen_params;
+ sd::VideoConfig processed_params = *sd_vid_gen_params;
if (!preprocessing.prepare_inputs(processed_params, request.width, request.height))
return false;
sd_vid_gen_params = &processed_params;
@@ -1897,7 +1898,6 @@ namespace sd::pipeline {
sd->rng->manual_seed(request.seed);
sd->sampler_rng->manual_seed(request.seed);
- sd->set_flow_shift(sd_vid_gen_params->sample_params.flow_shift);
if (!sd->apply_loras(sd_vid_gen_params->loras, sd_vid_gen_params->lora_count))
return false;
sd->reset_generation_extensions();
@@ -2141,13 +2141,10 @@ namespace sd::pipeline {
make_hires_sigma_schedule(sd,
request.hires,
request.sample_params,
- hires_sample_method,
plan.sample_steps,
sd->get_image_seq_len(hires_request.height, hires_request.width) * T,
&hires_scheduler_steps);
- float hires_eta = resolve_eta(sd,
- sd_vid_gen_params->sample_params.eta,
- hires_sample_method);
+ float hires_eta = plan.eta;
LOG_VERBOSE("sample(latent upscale) %dx%dx%d", W, H, T);
LOG_INFO("video latent spatial upscale refine: scheduler_steps=%d, denoising_strength=%.2f, sampler=%s, sigma_sched_size=%zu%s",
@@ -2197,11 +2194,11 @@ namespace sd::pipeline {
int64_t latent_end = ggml_time_ms();
LOG_INFO("generating latent video completed, taking %.2fs", (latent_end - latent_start) * 1.0f / 1000);
- sd_audio_t* generated_audio = nullptr;
+ sd::Audio* generated_audio = nullptr;
if (sd->version == VERSION_WAN2_2_S2V && sd_vid_gen_params->ref_audios_count > 0) {
// Return the driving track for muxing with the generated video.
- const sd_audio_t& driving = sd_vid_gen_params->ref_audios[0];
- generated_audio = (sd_audio_t*)malloc(sizeof(sd_audio_t));
+ const sd::Audio& driving = sd_vid_gen_params->ref_audios[0];
+ generated_audio = (sd::Audio*)malloc(sizeof(sd::Audio));
if (generated_audio != nullptr) {
generated_audio->sample_rate = driving.sample_rate;
generated_audio->channels = driving.channels;
@@ -2270,13 +2267,13 @@ namespace sd::pipeline {
if (sd->get_cancel_flag() == SD_CANCEL_ALL) {
LOG_ERROR("cancelling generation before video decode");
- free_sd_audio(generated_audio);
+ sd::free_audio(generated_audio);
return false;
}
auto result = decode_video_outputs(sd, latent_upscale_enabled ? hires_request : request, final_latent, num_frames_out);
LOG_DEBUG("decode_video_outputs returned %s", result == nullptr ? "nullptr (failed)" : "frames");
if (result == nullptr) {
- free_sd_audio(generated_audio);
+ sd::free_audio(generated_audio);
return false;
}
@@ -2305,14 +2302,14 @@ namespace sd::pipeline {
try {
*info_out = info;
} catch (...) {
- free_sd_audio(generated_audio);
+ sd::free_audio(generated_audio);
throw;
}
}
if (audio_out != nullptr) {
*audio_out = generated_audio;
} else {
- free_sd_audio(generated_audio);
+ sd::free_audio(generated_audio);
}
return true;
}
diff --git a/src/pipeline/video_upscale.cpp b/src/pipeline/video_upscale.cpp
index f9b631e..98837e8 100644
--- a/src/pipeline/video_upscale.cpp
+++ b/src/pipeline/video_upscale.cpp
@@ -1,4 +1,7 @@
#include "pipeline/video_upscale.h"
+#include "api/options.h"
+#include "api/result.h"
+#include "generation_info.h"
#include <climits>
@@ -66,7 +69,7 @@ namespace sd {
model_.reset();
}
- bool VideoUpscaler::init(const sd_video_upscaler_ctx_params_t& params) {
+ bool VideoUpscaler::init(const sd::VideoUpscalerConfig& params) {
if (!params.diffusion_model_path || !*params.diffusion_model_path ||
!params.vae_path || !*params.vae_path || params.upscale_factor <= 0) {
LOG_ERROR("video upscale requires diffusion and decoder model paths and a positive upscale factor");
@@ -110,7 +113,7 @@ namespace sd {
return model_->init(runtime_, weights, upscale_factor_);
}
- bool VideoUpscaler::upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t** result_out) {
+ bool VideoUpscaler::upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult** result_out) {
if (!result_out)
return false;
*result_out = nullptr;
@@ -130,11 +133,11 @@ namespace sd {
}
}
sd::ParallelScope scope(executor_.get());
- auto result = std::unique_ptr<sd_video_generation_result_t, decltype(&free_sd_video_generation_result)>(
- static_cast<sd_video_generation_result_t*>(calloc(1, sizeof(sd_video_generation_result_t))), free_sd_video_generation_result);
+ auto result = std::unique_ptr<sd::VideoGenerationResult, decltype(&sd::free_video_generation_result)>(
+ static_cast<sd::VideoGenerationResult*>(calloc(1, sizeof(sd::VideoGenerationResult))), sd::free_video_generation_result);
if (!result)
return false;
- result->frames = static_cast<sd_image_t*>(calloc(params.frame_count, sizeof(sd_image_t)));
+ result->frames = static_cast<sd::Image*>(calloc(params.frame_count, sizeof(sd::Image)));
if (!result->frames)
return false;
result->frame_count = params.frame_count;
@@ -148,55 +151,50 @@ namespace sd {
} // namespace pipeline
} // namespace sd
-struct video_upscaler_ctx_t {
+struct sd_vid_upscaler {
sd::pipeline::VideoUpscaler upscaler;
};
-void sd_video_upscaler_ctx_params_init(sd_video_upscaler_ctx_params_t* params) {
- if (!params)
- return;
- *params = {};
- params->n_threads = sd_get_num_physical_cores();
+sd_status_t sd_vid_upscaler_create(const sd_vid_upscaler_opt_t* opt, sd_vid_upscaler_t** out) {
+ if (out)
+ *out = nullptr;
+ return sd::api::call([&] {
+ if (!out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(opt, sd::api::VidUpscaler, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ auto result = std::make_unique<sd_vid_upscaler_t>();
+ if (!result->upscaler.init(config.post.video_upscaler))
+ return sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not load video upscaler");
+ *out = result.release();
+ return SD_STATUS_OK;
+ },
+ SD_STATUS_LOAD_FAILED);
}
-void sd_video_upscale_params_init(sd_video_upscale_params_t* params) {
- if (!params)
- return;
- *params = {};
- params->fps = 24;
- params->seed = 42;
- params->window_size = 16;
- params->temporal_chunk_size = 5;
+void sd_vid_upscaler_free(sd_vid_upscaler_t* upscaler) {
+ delete upscaler;
}
-video_upscaler_ctx_t* new_video_upscaler_ctx(const sd_video_upscaler_ctx_params_t* params) {
- if (!params)
- return nullptr;
- try {
- auto ctx = std::make_unique<video_upscaler_ctx_t>();
- if (!ctx->upscaler.init(*params))
- return nullptr;
- return ctx.release();
- } catch (const std::exception& error) {
- LOG_ERROR("video upscaler initialization failed: %s", error.what());
- return nullptr;
- }
-}
-
-bool upscale_video(video_upscaler_ctx_t* ctx, const sd_video_upscale_params_t* params, sd_video_generation_result_t** result_out) {
- if (!result_out)
- return false;
- *result_out = nullptr;
- if (!ctx || !params)
- return false;
- try {
- return ctx->upscaler.upscale(*params, result_out);
- } catch (const std::exception& error) {
- LOG_ERROR("video upscale failed: %s", error.what());
- return false;
- }
-}
-
-void free_video_upscaler_ctx(video_upscaler_ctx_t* ctx) {
- delete ctx;
+sd_status_t sd_upscale_video(sd_vid_upscaler_t* upscaler, const sd_vid_upscale_opt_t* opt, sd_result_t** out) {
+ if (out)
+ *out = nullptr;
+ return sd::api::call([&] {
+ if (!out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(opt, sd::api::VidUpscale, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ if (!upscaler)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required");
+ sd::VideoGenerationResult* video = nullptr;
+ if (!upscaler->upscaler.upscale(config.post.video_upscale, &video))
+ return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "video upscale failed");
+ return sd::make_video_result(video, out) ? SD_STATUS_OK
+ : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result");
+ },
+ SD_STATUS_GENERATION_FAILED);
}
diff --git a/src/pipeline/video_upscale.h b/src/pipeline/video_upscale.h
index ec5815c..51ab768 100644
--- a/src/pipeline/video_upscale.h
+++ b/src/pipeline/video_upscale.h
@@ -4,6 +4,8 @@
#include <map>
#include <memory>
+#include "api/result.h"
+#include "config.h"
#include "core/ggml_extend_backend.h"
#include "core/tensor.hpp"
#include "model_manager.h"
@@ -34,7 +36,7 @@ namespace sd {
virtual ~VideoUpscaleModel() = default;
virtual bool init(VideoUpscaleRuntime& runtime, const String2TensorStorage& weights, int upscale_factor) = 0;
virtual void end() = 0;
- virtual bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t* result) = 0;
+ virtual bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult* result) = 0;
};
bool is_relay_vsr_model(const String2TensorStorage& weights);
@@ -50,8 +52,8 @@ namespace sd {
public:
~VideoUpscaler();
- bool init(const sd_video_upscaler_ctx_params_t& params);
- bool upscale(const sd_video_upscale_params_t& params, sd_video_generation_result_t** result_out);
+ bool init(const sd::VideoUpscalerConfig& params);
+ bool upscale(const sd::VideoUpscaleConfig& params, sd::VideoGenerationResult** result_out);
};
} // namespace pipeline
diff --git a/src/runtime/denoiser.hpp b/src/runtime/denoiser.hpp
index 701d8d9..b947074 100644
--- a/src/runtime/denoiser.hpp
+++ b/src/runtime/denoiser.hpp
@@ -1,6 +1,8 @@
#ifndef __SD_RUNTIME_DENOISER_HPP__
#define __SD_RUNTIME_DENOISER_HPP__
+#include "core/option_types.h"
+
#include <algorithm>
#include <array>
#include <cctype>
diff --git a/src/runtime/image_preprocess.cpp b/src/runtime/image_preprocess.cpp
index 445f0d1..d5e17d9 100644
--- a/src/runtime/image_preprocess.cpp
+++ b/src/runtime/image_preprocess.cpp
@@ -196,7 +196,7 @@ namespace sd {
return options;
}
- bool ImagePreprocessor::validate_inputs(const sd_img_gen_params_t& params) const {
+ bool ImagePreprocessor::validate_inputs(const sd::ImageConfig& params) const {
const std::map<ImageTarget, int> counts = {
{ImageTarget::Init, params.init_image.data != nullptr},
{ImageTarget::Mask, params.mask_image.data != nullptr},
@@ -215,7 +215,7 @@ namespace sd {
return valid_;
}
- bool ImagePreprocessor::validate_inputs(const sd_vid_gen_params_t& params) const {
+ bool ImagePreprocessor::validate_inputs(const sd::VideoConfig& params) const {
const std::map<ImageTarget, int> counts = {
{ImageTarget::Init, params.init_image.data != nullptr},
{ImageTarget::End, params.end_image.data != nullptr},
@@ -370,7 +370,7 @@ namespace sd {
return apply_transform(image, options, p, label, default_filter);
}
- Tensor<float> ImagePreprocessor::preprocess_input(sd_image_t image, ImageTarget target, int index, int width, int height) {
+ Tensor<float> ImagePreprocessor::preprocess_input(sd::Image image, ImageTarget target, int index, int width, int height) {
if (image.data == nullptr || image.width == 0 || image.height == 0 || image.width > INT_MAX || image.height > INT_MAX || image.channel < 1 || image.channel > 4) {
fail(std::string(enum_name(target, image_targets)) + ": invalid input image");
return {};
@@ -419,7 +419,7 @@ namespace sd {
std::free(image.data);
}
- bool ImagePreprocessor::prepare_image(sd_image_t& image, ImageTarget target, int index, int width, int height) {
+ bool ImagePreprocessor::prepare_image(sd::Image& image, ImageTarget target, int index, int width, int height) {
if (image.data == nullptr)
return true;
auto options = resolve_options(target, index);
@@ -431,13 +431,13 @@ namespace sd {
if (output.data == nullptr)
return fail("could not allocate input preprocessing buffer");
owned_images_.push_back(output);
- if (canny && !preprocess_canny(output, 0.08f, 0.08f, 0.8f, 1.f, false))
+ if (canny && !sd::preprocess_canny(output, 0.08f, 0.08f, 0.8f, 1.f, false))
return fail("Canny preprocessing failed");
image = output;
return true;
}
- bool ImagePreprocessor::prepare_array(sd_image_t*& images, int count, ImageTarget target, std::vector<sd_image_t>& storage, int width, int height) {
+ bool ImagePreprocessor::prepare_array(sd::Image*& images, int count, ImageTarget target, std::vector<sd::Image>& storage, int width, int height) {
if (count < 0 || (count > 0 && images == nullptr))
return fail(std::string("invalid image array: ") + enum_name(target, image_targets));
if (count == 0)
@@ -453,7 +453,7 @@ namespace sd {
return true;
}
- bool ImagePreprocessor::prepare_inputs(sd_img_gen_params_t& params, int width, int height) {
+ bool ImagePreprocessor::prepare_inputs(sd::ImageConfig& params, int width, int height) {
if (prepared_)
return fail("inputs have already been prepared");
prepared_ = true;
@@ -470,7 +470,7 @@ namespace sd {
return true;
}
- bool ImagePreprocessor::prepare_inputs(sd_vid_gen_params_t& params, int width, int height) {
+ bool ImagePreprocessor::prepare_inputs(sd::VideoConfig& params, int width, int height) {
if (prepared_)
return fail("inputs have already been prepared");
prepared_ = true;
diff --git a/src/runtime/image_preprocess.h b/src/runtime/image_preprocess.h
index 7dfc458..078befd 100644
--- a/src/runtime/image_preprocess.h
+++ b/src/runtime/image_preprocess.h
@@ -1,11 +1,14 @@
#ifndef __SD_RUNTIME_IMAGE_PREPROCESS_H__
#define __SD_RUNTIME_IMAGE_PREPROCESS_H__
+#include "core/media.h"
+
#include <map>
#include <string>
#include <utility>
#include <vector>
+#include "config.h"
#include "core/tensor.hpp"
#include "stable-diffusion.h"
@@ -61,30 +64,30 @@ namespace sd {
ImageTransform init_transform_;
bool has_init_transform_ = false;
bool prepared_ = false;
- std::vector<sd_image_t> owned_images_;
- std::vector<sd_image_t> ref_images_;
- std::vector<sd_image_t> id_images_;
- std::vector<sd_image_t> control_frames_;
+ std::vector<sd::Image> owned_images_;
+ std::vector<sd::Image> ref_images_;
+ std::vector<sd::Image> id_images_;
+ std::vector<sd::Image> control_frames_;
bool fail(const std::string& message) const;
std::map<std::string, std::string> resolve_options(ImageTarget target, int index) const;
Tensor<float> apply_transform(const Tensor<float>& image, const std::map<std::string, std::string>& options, ImageTransform plan, const std::string& label, ops::InterpolateMode default_filter) const;
- bool prepare_image(sd_image_t& image, ImageTarget target, int index, int width, int height);
- bool prepare_array(sd_image_t*& images, int count, ImageTarget target, std::vector<sd_image_t>& storage, int width = -1, int height = -1);
+ bool prepare_image(sd::Image& image, ImageTarget target, int index, int width, int height);
+ bool prepare_array(sd::Image*& images, int count, ImageTarget target, std::vector<sd::Image>& storage, int width = -1, int height = -1);
public:
explicit ImagePreprocessor(const char* rules = nullptr);
~ImagePreprocessor();
ImagePreprocessor(const ImagePreprocessor&) = delete;
ImagePreprocessor& operator=(const ImagePreprocessor&) = delete;
- bool prepare_inputs(sd_img_gen_params_t& params, int width, int height);
- bool prepare_inputs(sd_vid_gen_params_t& params, int width, int height);
+ bool prepare_inputs(sd::ImageConfig& params, int width, int height);
+ bool prepare_inputs(sd::VideoConfig& params, int width, int height);
bool is_valid() const { return valid_; }
- bool validate_inputs(const sd_img_gen_params_t& params) const;
- bool validate_inputs(const sd_vid_gen_params_t& params) const;
+ bool validate_inputs(const sd::ImageConfig& params) const;
+ bool validate_inputs(const sd::VideoConfig& params) const;
Tensor<float> apply_geometry(const Tensor<float>& image, ImageTarget target, int index, int width, int height, ImageResizeMode default_mode = ImageResizeMode::Stretch, ops::InterpolateMode default_filter = ops::InterpolateMode::Nearest, ImageTransform* plan_out = nullptr) const;
- Tensor<float> preprocess_input(sd_image_t image, ImageTarget target, int index = 0, int width = -1, int height = -1);
+ Tensor<float> preprocess_input(sd::Image image, ImageTarget target, int index = 0, int width = -1, int height = -1);
};
} // namespace sd
diff --git a/src/runtime/preprocessing.hpp b/src/runtime/preprocessing.hpp
index ab4ca29..da91e46 100644
--- a/src/runtime/preprocessing.hpp
+++ b/src/runtime/preprocessing.hpp
@@ -1,6 +1,8 @@
#ifndef __SD_RUNTIME_PREPROCESSING_HPP__
#define __SD_RUNTIME_PREPROCESSING_HPP__
+#include "core/media.h"
+
#include <cmath>
#include <limits>
@@ -93,7 +95,7 @@ static inline void preprocessing_tensor_frame_to_sd_image(const sd::Tensor<float
}
}
-static inline sd::Tensor<float> sd_image_to_preprocessing_tensor(sd_image_t image) {
+static inline sd::Tensor<float> sd_image_to_preprocessing_tensor(sd::Image image) {
sd::Tensor<float> tensor({static_cast<int64_t>(image.width), static_cast<int64_t>(image.height), static_cast<int64_t>(image.channel), 1});
for (uint32_t y = 0; y < image.height; ++y) {
for (uint32_t x = 0; x < image.width; ++x) {
@@ -293,7 +295,7 @@ static inline void threshold_hystersis(sd::Tensor<float>* img, float high_thresh
}
}
-bool preprocess_canny(sd_image_t img, float high_threshold, float low_threshold, float weak, float strong, bool inverse) {
+bool sd::preprocess_canny(sd::Image img, float high_threshold, float low_threshold, float weak, float strong, bool inverse) {
float kX[9] = {
-1, 0, 1,
-2, 0, 2,
diff --git a/src/runtime/sample-cache.cpp b/src/runtime/sample-cache.cpp
index 0b92c73..c8ea2c1 100644
--- a/src/runtime/sample-cache.cpp
+++ b/src/runtime/sample-cache.cpp
@@ -1,8 +1,9 @@
#include "runtime/sample-cache.h"
+#include "config.h"
namespace sd_sample {
- static float get_cache_reuse_threshold(const sd_cache_params_t& params) {
+ static float get_cache_reuse_threshold(const sd::CacheConfig& params) {
float reuse_threshold = params.reuse_threshold;
if (reuse_threshold == INFINITY) {
if (params.mode == SD_CACHE_EASYCACHE) {
@@ -26,7 +27,7 @@ namespace sd_sample {
return mode == SampleCacheMode::CACHEDIT;
}
- static bool has_valid_cache_percent_range(const sd_cache_params_t& cache_params) {
+ static bool has_valid_cache_percent_range(const sd::CacheConfig& cache_params) {
if (cache_params.mode != SD_CACHE_EASYCACHE && cache_params.mode != SD_CACHE_UCACHE) {
return true;
}
@@ -40,7 +41,7 @@ namespace sd_sample {
static void init_easycache_runtime(SampleCacheRuntime& runtime,
SDVersion version,
- const sd_cache_params_t& cache_params,
+ const sd::CacheConfig& cache_params,
Denoiser* denoiser) {
if (!sd_version_is_dit(version)) {
LOG_WARN("EasyCache requested but not supported for this model type");
@@ -68,7 +69,7 @@ namespace sd_sample {
static void init_ucache_runtime(SampleCacheRuntime& runtime,
SDVersion version,
- const sd_cache_params_t& cache_params,
+ const sd::CacheConfig& cache_params,
Denoiser* denoiser,
const std::vector<float>& sigmas) {
if (!sd_version_is_unet(version)) {
@@ -104,7 +105,7 @@ namespace sd_sample {
static void init_cachedit_runtime(SampleCacheRuntime& runtime,
SDVersion version,
- const sd_cache_params_t& cache_params,
+ const sd::CacheConfig& cache_params,
const std::vector<float>& sigmas) {
if (!sd_version_is_dit(version)) {
LOG_WARN("CacheDIT requested but not supported for this model type (only DiT models)");
@@ -147,7 +148,7 @@ namespace sd_sample {
static void init_spectrum_runtime(SampleCacheRuntime& runtime,
SDVersion version,
- const sd_cache_params_t& cache_params,
+ const sd::CacheConfig& cache_params,
const std::vector<float>& sigmas) {
if (!sd_version_is_unet(version) && !sd_version_is_dit(version)) {
LOG_WARN("Spectrum requested but not supported for this model type (only UNET and DiT models)");
@@ -174,7 +175,7 @@ namespace sd_sample {
}
SampleCacheRuntime init_sample_cache_runtime(SDVersion version,
- const sd_cache_params_t* cache_params,
+ const sd::CacheConfig* cache_params,
Denoiser* denoiser,
const std::vector<float>& sigmas) {
SampleCacheRuntime runtime;
diff --git a/src/runtime/sample-cache.h b/src/runtime/sample-cache.h
index 30ff66e..fd22edc 100644
--- a/src/runtime/sample-cache.h
+++ b/src/runtime/sample-cache.h
@@ -1,6 +1,8 @@
#ifndef __SD_RUNTIME_SAMPLE_CACHE_H__
#define __SD_RUNTIME_SAMPLE_CACHE_H__
+#include "config.h"
+
#include <vector>
#include "core/tensor.hpp"
@@ -51,7 +53,7 @@ namespace sd_sample {
};
SampleCacheRuntime init_sample_cache_runtime(SDVersion version,
- const sd_cache_params_t* cache_params,
+ const sd::CacheConfig* cache_params,
Denoiser* denoiser,
const std::vector<float>& sigmas);
diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp
index df2cfcb..acc3672 100644
--- a/src/stable-diffusion.cpp
+++ b/src/stable-diffusion.cpp
@@ -1,901 +1,135 @@
-#include "stable-diffusion.h"
-#include "generation_info.h"
-
-#include <algorithm>
-#include <cinttypes>
-#include <cmath>
-#include <cstdlib>
-
-#include "core/ggml_extend.h"
-#include "core/util.h"
+#include "api/api.h"
+#include "api/options.h"
+#include "api/result.h"
#include "pipeline/diffusion_engine.h"
#include "pipeline/generation.h"
-#include "pipeline/request.h"
-
-#define NONE_STR "NONE"
-
-static float get_cache_reuse_threshold(const sd_cache_params_t& params) {
- float reuse_threshold = params.reuse_threshold;
- if (reuse_threshold == INFINITY) {
- if (params.mode == SD_CACHE_EASYCACHE) {
- reuse_threshold = 0.2f;
- } else if (params.mode == SD_CACHE_UCACHE) {
- reuse_threshold = 1.0f;
- }
- }
- return std::max(0.0f, reuse_threshold);
-}
-
-const char* sd_type_name(enum sd_type_t type) {
- if (type == SD_TYPE_F8_E4M3) {
- return "f8_e4m3";
- }
- if (type == SD_TYPE_F8_E5M2) {
- return "f8_e5m2";
- }
- const auto ggml_type = sd_type_to_ggml_type(type);
- if (ggml_type != GGML_TYPE_COUNT) {
- return ggml_type_name(ggml_type);
- }
- return NONE_STR;
-}
-
-enum sd_type_t str_to_sd_type(const char* str) {
- if (!strcmp(str, "f8_e4m3")) {
- return SD_TYPE_F8_E4M3;
- }
- if (!strcmp(str, "f8_e5m2")) {
- return SD_TYPE_F8_E5M2;
- }
- for (int i = 0; i < std::min<int>(SD_TYPE_COUNT, GGML_TYPE_COUNT); i++) {
- auto trait = ggml_get_type_traits((ggml_type)i);
- if (!strcmp(str, trait->type_name)) {
- return (enum sd_type_t)i;
- }
- }
- return SD_TYPE_COUNT;
-}
-
-const char* rng_type_to_str[] = {
- "std_default",
- "cuda",
- "cpu",
-};
-
-const char* sd_rng_type_name(enum rng_type_t rng_type) {
- if (rng_type < RNG_TYPE_COUNT) {
- return rng_type_to_str[rng_type];
- }
- return NONE_STR;
-}
-
-enum rng_type_t str_to_rng_type(const char* str) {
- for (int i = 0; i < RNG_TYPE_COUNT; i++) {
- if (!strcmp(str, rng_type_to_str[i])) {
- return (enum rng_type_t)i;
- }
- }
- return RNG_TYPE_COUNT;
-}
-
-const char* sample_method_to_str[] = {
- "euler",
- "euler_a",
- "heun",
- "dpm2",
- "dpm++2s_a",
- "dpm++2m",
- "dpm++2mv2",
- "ipndm",
- "ipndm_v",
- "lcm",
- "ddim_trailing",
- "tcd",
- "res_multistep",
- "res_2s",
- "er_sde",
- "euler_cfg_pp",
- "euler_a_cfg_pp",
- "euler_ge",
- "dpm++2m_sde",
- "dpm++2m_sde_bt",
- "lms",
-};
-
-static_assert(SAMPLE_METHOD_COUNT == sizeof(sample_method_to_str) / sizeof(sample_method_to_str[0]),
- "\nnumber of elements in sample_method_to_str[] != SAMPLE_METHOD_COUNT");
-
-const char* sd_sample_method_name(enum sample_method_t sample_method) {
- if (sample_method < SAMPLE_METHOD_COUNT) {
- return sample_method_to_str[sample_method];
- }
- return NONE_STR;
-}
-
-enum sample_method_t str_to_sample_method(const char* str) {
- for (int i = 0; i < SAMPLE_METHOD_COUNT; i++) {
- if (!strcmp(str, sample_method_to_str[i])) {
- return (enum sample_method_t)i;
- }
- }
- return SAMPLE_METHOD_COUNT;
-}
-
-const char* scheduler_to_str[] = {
- "discrete",
- "karras",
- "exponential",
- "ays",
- "gits",
- "sgm_uniform",
- "simple",
- "smoothstep",
- "kl_optimal",
- "lcm",
- "bong_tangent",
- "ltx2",
- "logit_normal",
- "flux2",
- "flux",
- "beta",
- "llada_image",
-};
-
-static_assert(SCHEDULER_COUNT == sizeof(scheduler_to_str) / sizeof(scheduler_to_str[0]),
- "\nnumber of elements in scheduler_to_str[] != SCHEDULER_COUNT");
-
-const char* sd_scheduler_name(enum scheduler_t scheduler) {
- if (scheduler < SCHEDULER_COUNT) {
- return scheduler_to_str[scheduler];
- }
- return NONE_STR;
-}
-
-enum scheduler_t str_to_scheduler(const char* str) {
- if (!strcmp(str, "normal")) {
- return DISCRETE_SCHEDULER;
- }
- for (int i = 0; i < SCHEDULER_COUNT; i++) {
- if (!strcmp(str, scheduler_to_str[i])) {
- return (enum scheduler_t)i;
- }
- }
- return SCHEDULER_COUNT;
-}
-
-const char* prediction_to_str[] = {
- "eps",
- "v",
- "edm_v",
- "sd3_flow",
- "flux_flow",
- "sefi_flow",
- "minit2i_flow",
- "sensenova_u1_flow",
- "flux3_flow",
- "h3_av_flow",
- "ming_image_flow",
-};
-
-const char* sd_prediction_name(enum prediction_t prediction) {
- if (prediction >= EPS_PRED && prediction < PREDICTION_COUNT) {
- return prediction_to_str[prediction];
- }
- return NONE_STR;
-}
-
-enum prediction_t str_to_prediction(const char* str) {
- for (int i = 0; i < PREDICTION_COUNT; i++) {
- if (!strcmp(str, prediction_to_str[i])) {
- return (enum prediction_t)i;
- }
- }
- return PREDICTION_COUNT;
-}
-
-const char* preview_to_str[] = {
- "none",
- "proj",
- "tae",
- "vae",
-};
-
-const char* sd_preview_name(enum preview_t preview) {
- if (preview < PREVIEW_COUNT) {
- return preview_to_str[preview];
- }
- return NONE_STR;
-}
-
-enum preview_t str_to_preview(const char* str) {
- for (int i = 0; i < PREVIEW_COUNT; i++) {
- if (!strcmp(str, preview_to_str[i])) {
- return (enum preview_t)i;
- }
- }
- return PREVIEW_COUNT;
-}
-
-const char* lora_apply_mode_to_str[] = {
- "auto",
- "immediately",
- "at_runtime",
-};
-
-const char* sd_lora_apply_mode_name(enum lora_apply_mode_t mode) {
- if (mode < LORA_APPLY_MODE_COUNT) {
- return lora_apply_mode_to_str[mode];
- }
- return NONE_STR;
-}
-
-enum lora_apply_mode_t str_to_lora_apply_mode(const char* str) {
- for (int i = 0; i < LORA_APPLY_MODE_COUNT; i++) {
- if (!strcmp(str, lora_apply_mode_to_str[i])) {
- return (enum lora_apply_mode_t)i;
- }
- }
- return LORA_APPLY_MODE_COUNT;
-}
-
-const char* hires_upscaler_to_str[] = {
- "None",
- "Latent",
- "Latent (nearest)",
- "Latent (nearest-exact)",
- "Latent (antialiased)",
- "Latent (bicubic)",
- "Latent (bicubic antialiased)",
- "Lanczos",
- "Nearest",
- "Model",
-};
-
-const char* sd_hires_upscaler_name(enum sd_hires_upscaler_t upscaler) {
- if (upscaler >= SD_HIRES_UPSCALER_NONE && upscaler < SD_HIRES_UPSCALER_COUNT) {
- return hires_upscaler_to_str[upscaler];
- }
- return NONE_STR;
-}
-
-enum sd_hires_upscaler_t str_to_sd_hires_upscaler(const char* str) {
- for (int i = 0; i < SD_HIRES_UPSCALER_COUNT; i++) {
- if (!strcmp(str, hires_upscaler_to_str[i])) {
- return (enum sd_hires_upscaler_t)i;
- }
- }
- return SD_HIRES_UPSCALER_COUNT;
-}
-
-const char* control_type_to_str[] = {
- "none",
- "openpose",
- "depth",
- "scribble",
- "canny",
- "normal",
- "segment",
- "tile",
- "repaint",
-};
-
-static_assert(SD_CONTROL_TYPE_COUNT == sizeof(control_type_to_str) / sizeof(control_type_to_str[0]),
- "\nnumber of elements in control_type_to_str[] != SD_CONTROL_TYPE_COUNT");
-const char* sd_control_type_name(enum sd_control_type_t control_type) {
- if (control_type >= SD_CONTROL_TYPE_NONE && control_type < SD_CONTROL_TYPE_COUNT) {
- return control_type_to_str[control_type];
- }
- return NONE_STR;
-}
-
-enum sd_control_type_t str_to_sd_control_type(const char* str) {
- for (int i = 0; i < SD_CONTROL_TYPE_COUNT; i++) {
- if (!strcmp(str, control_type_to_str[i])) {
- return (enum sd_control_type_t)i;
- }
- }
- static const struct {
- const char* alias;
- enum sd_control_type_t type;
- } aliases[] = {
- {"hed", SD_CONTROL_TYPE_SCRIBBLE},
- {"softedge", SD_CONTROL_TYPE_SCRIBBLE},
- {"pidi", SD_CONTROL_TYPE_SCRIBBLE},
- {"ted", SD_CONTROL_TYPE_SCRIBBLE},
- {"lineart", SD_CONTROL_TYPE_CANNY},
- {"anime_lineart", SD_CONTROL_TYPE_CANNY},
- {"mlsd", SD_CONTROL_TYPE_CANNY},
- {"seg", SD_CONTROL_TYPE_SEGMENT},
- {"inpaint", SD_CONTROL_TYPE_REPAINT},
- {"outpaint", SD_CONTROL_TYPE_REPAINT},
- };
- for (const auto& entry : aliases) {
- if (!strcmp(str, entry.alias)) {
- return entry.type;
- }
- }
- return SD_CONTROL_TYPE_COUNT;
-}
-
-const char* sd_vae_format_name(enum sd_vae_format_t format) {
- switch (format) {
- case SD_VAE_FORMAT_AUTO:
- return "auto";
- case SD_VAE_FORMAT_FLUX:
- return "flux";
- case SD_VAE_FORMAT_SD3:
- return "sd3";
- case SD_VAE_FORMAT_FLUX2:
- return "flux2";
- case SD_VAE_FORMAT_WAN:
- return "wan";
- default:
- return NONE_STR;
- }
-}
-
-void sd_cache_params_init(sd_cache_params_t* cache_params) {
- *cache_params = {};
- cache_params->mode = SD_CACHE_DISABLED;
- cache_params->reuse_threshold = INFINITY;
- cache_params->start_percent = 0.15f;
- cache_params->end_percent = 0.95f;
- cache_params->error_decay_rate = 1.0f;
- cache_params->use_relative_threshold = true;
- cache_params->reset_error_on_compute = true;
- cache_params->Fn_compute_blocks = 8;
- cache_params->Bn_compute_blocks = 0;
- cache_params->residual_diff_threshold = 0.08f;
- cache_params->max_warmup_steps = 8;
- cache_params->max_cached_steps = -1;
- cache_params->max_continuous_cached_steps = -1;
- cache_params->taylorseer_n_derivatives = 1;
- cache_params->taylorseer_skip_interval = 1;
- cache_params->scm_mask = nullptr;
- cache_params->scm_policy_dynamic = true;
- cache_params->spectrum_w = 0.40f;
- cache_params->spectrum_m = 3;
- cache_params->spectrum_lam = 1.0f;
- cache_params->spectrum_window_size = 2;
- cache_params->spectrum_flex_window = 0.50f;
- cache_params->spectrum_warmup_steps = 4;
- cache_params->spectrum_stop_percent = 0.9f;
-}
-
-void sd_hires_params_init(sd_hires_params_t* hires_params) {
- *hires_params = {};
- hires_params->enabled = false;
- hires_params->upscaler = SD_HIRES_UPSCALER_LATENT;
- hires_params->model_path = nullptr;
- hires_params->scale = 2.0f;
- hires_params->target_width = 0;
- hires_params->target_height = 0;
- hires_params->steps = 0;
- hires_params->denoising_strength = 0.7f;
- hires_params->upscale_tile_size = 128;
- hires_params->custom_sigmas = nullptr;
- hires_params->custom_sigmas_count = 0;
-}
-
-void sd_ctx_params_init(sd_ctx_params_t* sd_ctx_params) {
- *sd_ctx_params = {};
- sd_ctx_params->n_threads = sd_get_num_physical_cores();
- sd_ctx_params->conditioning_cache_size = 4;
- sd_ctx_params->wtype = SD_TYPE_COUNT;
- sd_ctx_params->rng_type = CUDA_RNG;
- sd_ctx_params->sampler_rng_type = RNG_TYPE_COUNT;
- sd_ctx_params->prediction = PREDICTION_COUNT;
- sd_ctx_params->lora_apply_mode = LORA_APPLY_AUTO;
- sd_ctx_params->max_vram = nullptr;
- sd_ctx_params->disable_prefetch = false;
- sd_ctx_params->disable_segmented_compute = false;
- sd_ctx_params->batched_cfg = true;
- sd_ctx_params->eager_load = false;
- sd_ctx_params->enable_mmap = false;
- sd_ctx_params->flash_attn_mode = nullptr;
- sd_ctx_params->conv_direct_mode = nullptr;
- sd_ctx_params->diffusion_flash_attn = false;
- sd_ctx_params->sage_attn = false;
- sd_ctx_params->linear_scale = 0.f;
- sd_ctx_params->attn_scale = 0.f;
- sd_ctx_params->vae_format = SD_VAE_FORMAT_AUTO;
- sd_ctx_params->backend = nullptr;
- sd_ctx_params->params_backend = nullptr;
- sd_ctx_params->split_mode = nullptr;
- sd_ctx_params->auto_fit = true;
- sd_ctx_params->rpc_servers = nullptr;
- sd_ctx_params->model_args = nullptr;
- sd_ctx_params->pulid_weights_path = nullptr;
-}
-
-char* sd_ctx_params_to_str(const sd_ctx_params_t* sd_ctx_params) {
- char* buf = (char*)malloc(8192);
- if (!buf)
- return nullptr;
- buf[0] = '\0';
-
- snprintf(buf + strlen(buf), 8192 - strlen(buf),
- "model_path: %s\n"
- "clip_l_path: %s\n"
- "clip_g_path: %s\n"
- "clip_vision_path: %s\n"
- "t5xxl_path: %s\n"
- "llm_path: %s\n"
- "llm_vision_path: %s\n"
- "tokenizer: %s\n"
- "diffusion_model_path: %s\n"
- "high_noise_diffusion_model_path: %s\n"
- "uncond_diffusion_model_path: %s\n"
- "embeddings_connectors_path: %s\n"
- "vae_path: %s\n"
- "audio_vae_path: %s\n"
- "audio_encoder_path: %s\n"
- "taesd_path: %s\n"
- "control_net_path: %s\n"
- "photo_maker_path: %s\n"
- "pulid_weights_path: %s\n"
- "tensor_type_rules: %s\n"
- "n_threads: %d\n"
- "conditioning_cache_size: %d\n"
- "wtype: %s\n"
- "rng_type: %s\n"
- "sampler_rng_type: %s\n"
- "prediction: %s\n"
- "max_vram: %s\n"
- "disable_prefetch: %s\n"
- "disable_segmented_compute: %s\n"
- "eager_load: %s\n"
- "backend: %s\n"
- "params_backend: %s\n"
- "split_mode: %s\n"
- "model_args: %s\n"
- "auto_fit: %s\n"
- "flash_attn: %s\n"
- "flash_attn_mode: %s\n"
- "conv_direct_mode: %s\n"
- "diffusion_flash_attn: %s\n"
- "sage_attn: %s\n"
- "linear_scale: %g\n"
- "attn_scale: %g\n"
- "vae_format: %s\n",
- SAFE_STR(sd_ctx_params->model_path),
- SAFE_STR(sd_ctx_params->clip_l_path),
- SAFE_STR(sd_ctx_params->clip_g_path),
- SAFE_STR(sd_ctx_params->clip_vision_path),
- SAFE_STR(sd_ctx_params->t5xxl_path),
- SAFE_STR(sd_ctx_params->llm_path),
- SAFE_STR(sd_ctx_params->llm_vision_path),
- SAFE_STR(sd_ctx_params->tokenizer),
- SAFE_STR(sd_ctx_params->diffusion_model_path),
- SAFE_STR(sd_ctx_params->high_noise_diffusion_model_path),
- SAFE_STR(sd_ctx_params->uncond_diffusion_model_path),
- SAFE_STR(sd_ctx_params->embeddings_connectors_path),
- SAFE_STR(sd_ctx_params->vae_path),
- SAFE_STR(sd_ctx_params->audio_vae_path),
- SAFE_STR(sd_ctx_params->audio_encoder_path),
- SAFE_STR(sd_ctx_params->taesd_path),
- SAFE_STR(sd_ctx_params->control_net_path),
- SAFE_STR(sd_ctx_params->photo_maker_path),
- SAFE_STR(sd_ctx_params->pulid_weights_path),
- SAFE_STR(sd_ctx_params->tensor_type_rules),
- sd_ctx_params->n_threads,
- sd_ctx_params->conditioning_cache_size,
- sd_type_name(sd_ctx_params->wtype),
- sd_rng_type_name(sd_ctx_params->rng_type),
- sd_rng_type_name(sd_ctx_params->sampler_rng_type),
- sd_prediction_name(sd_ctx_params->prediction),
- SAFE_STR(sd_ctx_params->max_vram),
- BOOL_STR(sd_ctx_params->disable_prefetch),
- BOOL_STR(sd_ctx_params->disable_segmented_compute),
- BOOL_STR(sd_ctx_params->eager_load),
- SAFE_STR(sd_ctx_params->backend),
- SAFE_STR(sd_ctx_params->params_backend),
- SAFE_STR(sd_ctx_params->split_mode),
- SAFE_STR(sd_ctx_params->model_args),
- BOOL_STR(sd_ctx_params->auto_fit),
- BOOL_STR(sd_ctx_params->flash_attn),
- sd_ctx_params->flash_attn_mode && sd_ctx_params->flash_attn_mode[0] ? sd_ctx_params->flash_attn_mode : "auto (legacy flags honored)",
- sd_ctx_params->conv_direct_mode && sd_ctx_params->conv_direct_mode[0] ? sd_ctx_params->conv_direct_mode : "auto (legacy flags honored)",
- BOOL_STR(sd_ctx_params->diffusion_flash_attn),
- BOOL_STR(sd_ctx_params->sage_attn),
- sd_ctx_params->linear_scale,
- sd_ctx_params->attn_scale,
- sd_vae_format_name(sd_ctx_params->vae_format));
-
- return buf;
-}
-
-void sd_sample_params_init(sd_sample_params_t* sample_params) {
- *sample_params = {};
- sample_params->guidance.txt_cfg = 7.0f;
- sample_params->guidance.img_cfg = INFINITY;
- sample_params->guidance.distilled_guidance = 3.5f;
- sample_params->guidance.slg.layer_count = 0;
- sample_params->guidance.slg.layer_start = 0.01f;
- sample_params->guidance.slg.layer_end = 0.2f;
- sample_params->guidance.slg.scale = 0.f;
- sample_params->scheduler = SCHEDULER_COUNT;
- sample_params->sample_method = SAMPLE_METHOD_COUNT;
- sample_params->sample_steps = 20;
- sample_params->eta = INFINITY;
- sample_params->custom_sigmas = nullptr;
- sample_params->custom_sigmas_count = 0;
- sample_params->flow_shift = INFINITY;
- sample_params->extra_sample_args = nullptr;
-}
-
-char* sd_sample_params_to_str(const sd_sample_params_t* sample_params) {
- char* buf = (char*)malloc(4096);
- if (!buf)
- return nullptr;
- buf[0] = '\0';
-
- snprintf(buf + strlen(buf), 4096 - strlen(buf),
- "(txt_cfg: %.2f, "
- "img_cfg: %.2f, "
- "distilled_guidance: %.2f, "
- "slg.layer_count: %zu, "
- "slg.layer_start: %.2f, "
- "slg.layer_end: %.2f, "
- "slg.scale: %.2f, "
- "scheduler: %s, "
- "sample_method: %s, "
- "sample_steps: %d, "
- "eta: %.2f, "
- "shifted_timestep: %d, "
- "flow_shift: %.2f, "
- "extra_sample_args: %s)",
- sample_params->guidance.txt_cfg,
- std::isfinite(sample_params->guidance.img_cfg)
- ? sample_params->guidance.img_cfg
- : sample_params->guidance.txt_cfg,
- sample_params->guidance.distilled_guidance,
- sample_params->guidance.slg.layer_count,
- sample_params->guidance.slg.layer_start,
- sample_params->guidance.slg.layer_end,
- sample_params->guidance.slg.scale,
- sd_scheduler_name(sample_params->scheduler),
- sd_sample_method_name(sample_params->sample_method),
- sample_params->sample_steps,
- sample_params->eta,
- sample_params->shifted_timestep,
- sample_params->flow_shift,
- SAFE_STR(sample_params->extra_sample_args));
-
- return buf;
-}
-
-void sd_img_gen_params_init(sd_img_gen_params_t* sd_img_gen_params) {
- *sd_img_gen_params = {};
- sd_sample_params_init(&sd_img_gen_params->sample_params);
- sd_img_gen_params->clip_skip = -1;
- sd_img_gen_params->ref_images_count = 0;
- sd_img_gen_params->ref_image_args = "";
- sd_img_gen_params->width = 512;
- sd_img_gen_params->height = 512;
- sd_img_gen_params->strength = 0.75f;
- sd_img_gen_params->seed = -1;
- sd_img_gen_params->batch_count = 1;
- sd_img_gen_params->control_strength = 0.9f;
- sd_img_gen_params->ip_adapter_strength = 1.0f;
- sd_img_gen_params->qwen_image_layers = 3;
- sd_img_gen_params->circular_x = false;
- sd_img_gen_params->circular_y = false;
- sd_img_gen_params->pm_params = {nullptr, 0, nullptr, 20.f};
- sd_img_gen_params->pulid_params = {nullptr, 1.0f};
- sd_img_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
- sd_cache_params_init(&sd_img_gen_params->cache);
- sd_hires_params_init(&sd_img_gen_params->hires);
-}
-
-char* sd_img_gen_params_to_str(const sd_img_gen_params_t* sd_img_gen_params) {
- char* buf = (char*)malloc(4096);
- if (!buf)
- return nullptr;
- buf[0] = '\0';
-
- char* sample_params_str = sd_sample_params_to_str(&sd_img_gen_params->sample_params);
-
- snprintf(buf + strlen(buf), 4096 - strlen(buf),
- "prompt: %s\n"
- "negative_prompt: %s\n"
- "clip_skip: %d\n"
- "width: %d\n"
- "height: %d\n"
- "sample_params: %s\n"
- "strength: %.2f\n"
- "seed: %" PRId64
- "\n"
- "batch_count: %d\n"
- "qwen_image_layers: %d\n"
- "ref_images_count: %d\n"
- "ref_image_args: %s\n"
- "control_strength: %.2f\n"
- "control_type: %s\n"
- "photo maker: {style_strength = %.2f, id_images_count = %d, id_embed_path = %s}\n"
- "VAE tiling: %s (temporal=%s, extra_tiling_args=%s)\n"
- "circular_x: %s\n"
- "circular_y: %s\n"
- "hires: {enabled=%s, upscaler=%s, model_path=%s, scale=%.2f, target=%dx%d, steps=%d, denoising_strength=%.2f}\n",
- SAFE_STR(sd_img_gen_params->prompt),
- SAFE_STR(sd_img_gen_params->negative_prompt),
- sd_img_gen_params->clip_skip,
- sd_img_gen_params->width,
- sd_img_gen_params->height,
- SAFE_STR(sample_params_str),
- sd_img_gen_params->strength,
- sd_img_gen_params->seed,
- sd_img_gen_params->batch_count,
- sd_img_gen_params->qwen_image_layers,
- sd_img_gen_params->ref_images_count,
- SAFE_STR(sd_img_gen_params->ref_image_args),
- sd_img_gen_params->control_strength,
- sd_control_type_name(sd_img_gen_params->control_type),
- sd_img_gen_params->pm_params.style_strength,
- sd_img_gen_params->pm_params.id_images_count,
- SAFE_STR(sd_img_gen_params->pm_params.id_embed_path),
- BOOL_STR(sd_img_gen_params->vae_tiling_params.enabled),
- BOOL_STR(sd_img_gen_params->vae_tiling_params.temporal_tiling),
- SAFE_STR(sd_img_gen_params->vae_tiling_params.extra_tiling_args),
- BOOL_STR(sd_img_gen_params->circular_x),
- BOOL_STR(sd_img_gen_params->circular_y),
- BOOL_STR(sd_img_gen_params->hires.enabled),
- sd_hires_upscaler_name(sd_img_gen_params->hires.upscaler),
- SAFE_STR(sd_img_gen_params->hires.model_path),
- sd_img_gen_params->hires.scale,
- sd_img_gen_params->hires.target_width,
- sd_img_gen_params->hires.target_height,
- sd_img_gen_params->hires.steps,
- sd_img_gen_params->hires.denoising_strength);
- const char* cache_mode_str = "disabled";
- if (sd_img_gen_params->cache.mode == SD_CACHE_EASYCACHE) {
- cache_mode_str = "easycache";
- } else if (sd_img_gen_params->cache.mode == SD_CACHE_UCACHE) {
- cache_mode_str = "ucache";
- }
- snprintf(buf + strlen(buf), 4096 - strlen(buf),
- "cache: %s (threshold=%.3f, start=%.2f, end=%.2f)\n",
- cache_mode_str,
- get_cache_reuse_threshold(sd_img_gen_params->cache),
- sd_img_gen_params->cache.start_percent,
- sd_img_gen_params->cache.end_percent);
- free(sample_params_str);
- return buf;
-}
-
-void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params) {
- *sd_vid_gen_params = {};
- sd_sample_params_init(&sd_vid_gen_params->sample_params);
- sd_sample_params_init(&sd_vid_gen_params->high_noise_sample_params);
- sd_vid_gen_params->high_noise_sample_params.sample_steps = -1;
- sd_vid_gen_params->width = 512;
- sd_vid_gen_params->height = 512;
- sd_vid_gen_params->strength = 0.75f;
- sd_vid_gen_params->seed = -1;
- sd_vid_gen_params->video_frames = 6;
- sd_vid_gen_params->fps = 16;
- sd_vid_gen_params->moe_boundary = 0.875f;
- sd_vid_gen_params->vace_strength = 1.f;
- sd_vid_gen_params->vae_tiling_params = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
- sd_vid_gen_params->hires.enabled = false;
- sd_vid_gen_params->hires.upscaler = SD_HIRES_UPSCALER_LATENT;
- sd_vid_gen_params->hires.scale = 2.f;
- sd_vid_gen_params->hires.target_width = 0;
- sd_vid_gen_params->hires.target_height = 0;
- sd_vid_gen_params->hires.steps = 0;
- sd_vid_gen_params->hires.denoising_strength = 0.7f;
- sd_vid_gen_params->hires.upscale_tile_size = 128;
- sd_vid_gen_params->hires.custom_sigmas = nullptr;
- sd_vid_gen_params->hires.custom_sigmas_count = 0;
- sd_vid_gen_params->circular_x = false;
- sd_vid_gen_params->circular_y = false;
- sd_vid_gen_params->ref_image_args = "";
- sd_cache_params_init(&sd_vid_gen_params->cache);
-}
-
-struct sd_ctx_t {
- StableDiffusionGGML* sd = nullptr;
-};
-
-void sd_set_prompt_image_callback(sd_ctx_t* ctx, sd_prompt_image_cb_t cb, void* data) {
- if (ctx == nullptr || ctx->sd == nullptr) {
- return;
- }
- std::lock_guard<std::recursive_mutex> lock(ctx->sd->execution_mutex);
- if (!ctx->sd->executing_) {
- ctx->sd->prompt_image_callback = cb;
- ctx->sd->prompt_image_callback_data = data;
- }
-}
-
-void sd_set_generation_progress_callback(sd_ctx_t* ctx, sd_generation_progress_cb_t cb, void* data) {
- if (ctx == nullptr || ctx->sd == nullptr) {
- return;
- }
- std::lock_guard<std::recursive_mutex> lock(ctx->sd->execution_mutex);
- if (!ctx->sd->executing_) {
- ctx->sd->generation_progress.set_callback(cb, data);
- }
-}
-
-sd_ctx_t* new_sd_ctx(const sd_ctx_params_t* sd_ctx_params) {
- sd_ctx_t* sd_ctx = (sd_ctx_t*)malloc(sizeof(sd_ctx_t));
- if (sd_ctx == nullptr) {
- return nullptr;
- }
-
- sd_ctx->sd = new StableDiffusionGGML();
- if (sd_ctx->sd == nullptr) {
- free(sd_ctx);
- return nullptr;
- }
-
- if (!sd_ctx->sd->init(sd_ctx_params)) {
- delete sd_ctx->sd;
- sd_ctx->sd = nullptr;
- free(sd_ctx);
- return nullptr;
- }
- return sd_ctx;
-}
-
-void free_sd_ctx(sd_ctx_t* sd_ctx) {
- if (sd_ctx->sd != nullptr) {
- delete sd_ctx->sd;
- sd_ctx->sd = nullptr;
- }
- free(sd_ctx);
-}
-
-SD_API void sd_cancel_generation(sd_ctx_t* sd_ctx, enum sd_cancel_mode_t mode) {
- if (sd_ctx && sd_ctx->sd) {
- if (mode < SD_CANCEL_ALL || mode > SD_CANCEL_RESET) {
- mode = SD_CANCEL_ALL;
- }
- sd_ctx->sd->set_cancel_flag(mode);
- }
-}
-
-void free_sd_audio(sd_audio_t* audio) {
- if (audio == nullptr) {
- return;
- }
- free(audio->data);
- audio->data = nullptr;
- free(audio);
-}
-
-SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
- return false;
- }
- return sd_version_supports_image_generation(sd_ctx->sd->version);
-}
-
-SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
- return false;
- }
- if (sd_ctx->sd->config_->animatediff_loaded && sd_version_supports_animatediff(sd_ctx->sd->version)) {
- return true;
- }
- return sd_version_supports_video_generation(sd_ctx->sd->version);
-}
-
-SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) {
- return false;
- }
- return sd_ctx->sd->load_control_net_from_file(path);
-}
-
-SD_API bool sd_ctx_unload_control_net(sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
- return false;
- }
- return sd_ctx->sd->unload_control_net();
-}
-
-SD_API bool sd_ctx_has_control_net(const sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr) {
- return false;
- }
- if (sd_ctx->sd->control_net != nullptr) {
- return true;
- }
- // z-image keeps control net weights inside the diffusion runner, registered
- // under ModelComponent::ControlNet instead of a separate control_net runner.
- return sd_ctx->sd->model_manager != nullptr &&
- sd_ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0;
-}
-
-const char* sd_get_model_version_name(const sd_ctx_t* sd_ctx) {
- if (sd_ctx == nullptr || sd_ctx->sd == nullptr || sd_ctx->sd->version >= VERSION_COUNT) {
- return "Unknown";
- }
- return sd_version_name(sd_ctx->sd->version);
-}
-
-enum sample_method_t sd_get_default_sample_method(const sd_ctx_t* sd_ctx) {
- return sd::pipeline::default_sample_method(sd_ctx != nullptr ? sd_ctx->sd : nullptr);
-}
-
-enum scheduler_t sd_get_default_scheduler(const sd_ctx_t* sd_ctx, enum sample_method_t sample_method) {
- return sd::pipeline::default_scheduler(sd_ctx != nullptr ? sd_ctx->sd : nullptr, sample_method);
-}
-
-SD_API bool generate_image(sd_ctx_t* sd_ctx,
- const sd_img_gen_params_t* params,
- sd_image_generation_result_t** result_out) {
+#include <cmath>
+#include <cstdlib>
+#include <memory>
+#include <vector>
+
+static sd_status_t validate_generation(StableDiffusionGGML* engine, const sd::CacheConfig& cache, bool video) {
+ if (video ? !sd_version_supports_video_generation(engine->version) &&
+ !(engine->config_->animatediff_loaded && sd_version_supports_animatediff(engine->version))
+ : !sd_version_supports_image_generation(engine->version))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, nullptr, "model does not support this operation");
+ if (cache.mode == SD_CACHE_UCACHE && !sd_version_is_unet(engine->version))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "ucache requires a UNet model");
+ if ((cache.mode == SD_CACHE_EASYCACHE || cache.mode == SD_CACHE_DBCACHE ||
+ cache.mode == SD_CACHE_TAYLORSEER || cache.mode == SD_CACHE_CACHE_DIT) &&
+ !sd_version_is_dit(engine->version))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "selected cache requires a DiT model");
+ if (cache.mode == SD_CACHE_SPECTRUM && !sd_version_is_unet(engine->version) && !sd_version_is_dit(engine->version))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "cache.mode", "spectrum requires a UNet or DiT model");
+ return SD_STATUS_OK;
+}
+
+static sd_status_t generation_failure(StableDiffusionGGML* engine) {
+ if (engine->get_cancel_flag() == SD_CANCEL_ALL)
+ return sd::api::error(SD_STATUS_CANCELLED, nullptr, "generation cancelled");
+ return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "generation failed; see log for details");
+}
+
+static sd_status_t run_image(sd_ctx_t* ctx, const sd::ImageConfig* params, sd::ImageGenerationResult** result_out, const sd_opt_t* keys = nullptr) {
if (result_out)
*result_out = nullptr;
- if (!result_out || !sd_ctx || !sd_ctx->sd || !params)
- return false;
- StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd);
+ if (!result_out || !ctx || !ctx->sd || !params)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context, params and result_out are required");
+ StableDiffusionGGML::ExecutionScope execution(*ctx->sd);
if (!execution.ready)
- return false;
- sd_image_t* images = nullptr;
- int count = 0;
+ return execution.operation.acquired
+ ? sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not refresh model sources")
+ : sd::api::error(SD_STATUS_BUSY, nullptr, "context is already executing");
+ auto status = validate_generation(ctx->sd, params->cache, false);
+ if (status != SD_STATUS_OK)
+ return status;
+ if (sd::api::is_set(keys, "qwen_image_layers") && ctx->sd->version != VERSION_QWEN_IMAGE_LAYERED)
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "qwen_image_layers", "requires Qwen Image Layered");
+ const bool native_control = sd_version_is_control(ctx->sd->version) || ctx->sd->version == VERSION_FLEX_2 ||
+ (sd_version_is_z_image(ctx->sd->version) && ctx->sd->model_manager &&
+ ctx->sd->model_manager->registered_params_size({ModelComponent::ControlNet}) > 0);
+ if (params->control_image.data && !ctx->sd->control_net && !native_control)
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "control_image", "model has no active control-image component");
+ if (params->ip_adapter_image.data && !ctx->sd->ip_adapter)
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "ip_adapter_image", "no IP-Adapter is loaded");
+ if (params->pm_params.id_images_count > 0 && !ctx->sd->config_->photomaker_source_available)
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "photomaker.id_images", "no PhotoMaker component is loaded");
+ if (params->pulid_params.id_embedding_path && *params->pulid_params.id_embedding_path &&
+ (ctx->sd->version != VERSION_FLUX || !ctx->sd->config_->params.pulid_weights_path ||
+ !*ctx->sd->config_->params.pulid_weights_path))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "pulid.id_embedding_path", "requires a Flux model with PuLID weights");
+ if (params->sample_params.guidance.img_cfg != INFINITY && params->sample_params.guidance.img_cfg != 1.f &&
+ !sd_version_is_inpaint_or_unet_edit(ctx->sd->version) &&
+ !(params->ref_images_count > 0 && sd_version_supports_ref_latent_img_cfg(ctx->sd->version)))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "sample.guidance.img_cfg", "model does not support image CFG for these inputs");
+ sd::Image* images = nullptr;
+ int count = 0;
std::vector<nlohmann::json> infos;
try {
- if (!sd::pipeline::generate_image(sd_ctx->sd, params, &images, &count, &infos)) {
- free_sd_images(images, count);
- return false;
+ if (!sd::pipeline::generate_image(ctx->sd, params, &images, &count, &infos)) {
+ sd::free_images(images, count);
+ return generation_failure(ctx->sd);
}
- } catch (const std::exception& e) {
- LOG_ERROR("image generation failed: %s", e.what());
- free_sd_images(images, count);
- return false;
+ } catch (...) {
+ sd::free_images(images, count);
+ throw;
}
- return sd::make_image_generation_result(images, count, infos, result_out);
+ return sd::make_image_generation_result(images, count, infos, result_out) ? SD_STATUS_OK
+ : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate image result");
}
-SD_API bool generate_video(sd_ctx_t* sd_ctx,
- const sd_vid_gen_params_t* params,
- sd_video_generation_result_t** result_out) {
+static sd_status_t run_video(sd_ctx_t* ctx, const sd::VideoConfig* params, sd::VideoGenerationResult** result_out, const sd_opt_t* keys = nullptr) {
if (result_out)
*result_out = nullptr;
- if (!result_out || !sd_ctx || !sd_ctx->sd || !params)
- return false;
- StableDiffusionGGML::ExecutionScope execution(*sd_ctx->sd);
+ if (!result_out || !ctx || !ctx->sd || !params)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "context, params and result_out are required");
+ StableDiffusionGGML::ExecutionScope execution(*ctx->sd);
if (!execution.ready)
- return false;
- auto* result = static_cast<sd_video_generation_result_t*>(calloc(1, sizeof(sd_video_generation_result_t)));
+ return execution.operation.acquired
+ ? sd::api::error(SD_STATUS_LOAD_FAILED, nullptr, "could not refresh model sources")
+ : sd::api::error(SD_STATUS_BUSY, nullptr, "context is already executing");
+ auto status = validate_generation(ctx->sd, params->cache, true);
+ if (status != SD_STATUS_OK)
+ return status;
+ const bool vace = ctx->sd->diffusion_model &&
+ (ctx->sd->diffusion_model->get_desc() == "Wan2.1-VACE-1.3B" ||
+ ctx->sd->diffusion_model->get_desc() == "Wan2.x-VACE-14B");
+ if (params->control_frames_size > 0 && !vace)
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "control_frames", "requires a VACE model");
+ if (params->ref_videos_count > 0 && !sd_version_is_minimax_h3(ctx->sd->version))
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "ref_videos", "requires a MiniMax-H3 model");
+ if (params->ref_audios_count > 0 && !sd_version_is_minimax_h3(ctx->sd->version) && ctx->sd->version != VERSION_WAN2_2_S2V)
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "ref_audios", "requires MiniMax-H3 or Wan2.2 S2V");
+ if (ctx->sd->version == VERSION_WAN2_2_S2V) {
+ if (params->end_image.data)
+ return sd::api::error(SD_STATUS_UNSUPPORTED, "end_image", "Wan2.2 S2V does not support an end image");
+ if (params->ref_audios_count > 1 || (params->ref_audios_count && !ctx->sd->audio_encoder))
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, "ref_audios", "Wan2.2 S2V requires one driving track and an audio encoder");
+ }
+ auto result = std::unique_ptr<sd::VideoGenerationResult, decltype(&sd::free_video_generation_result)>(
+ static_cast<sd::VideoGenerationResult*>(calloc(1, sizeof(sd::VideoGenerationResult))), sd::free_video_generation_result);
if (!result)
- return false;
- try {
- nlohmann::json info;
- if (!sd::pipeline::generate_video(sd_ctx->sd, params, &result->frames, &result->frame_count,
- &result->audio, &result->fps, &info)) {
- free_sd_video_generation_result(result);
- return false;
- }
- result->generation_info = sd::copy_generation_info(info);
- if (!result->generation_info) {
- free_sd_video_generation_result(result);
- return false;
- }
- } catch (const std::exception& e) {
- LOG_ERROR("video generation failed: %s", e.what());
- free_sd_video_generation_result(result);
- return false;
- }
- *result_out = result;
- return true;
+ return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate video result");
+ nlohmann::json info;
+ if (!sd::pipeline::generate_video(ctx->sd, params, &result->frames, &result->frame_count,
+ &result->audio, &result->fps, &info))
+ return generation_failure(ctx->sd);
+ result->generation_info = sd::copy_generation_info(info);
+ if (!result->generation_info)
+ return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate video metadata");
+ *result_out = result.release();
+ return SD_STATUS_OK;
+}
+
+sd_status_t sd::generate_image(sd_ctx_t* ctx, const ImageConfig& config, ImageGenerationResult** result_out, const sd_opt_t* source) noexcept {
+ if (result_out)
+ *result_out = nullptr;
+ return sd::api::call([&] { return run_image(ctx, &config, result_out, source); }, SD_STATUS_GENERATION_FAILED);
}
-SD_API void free_sd_images(sd_image_t* result_images, int num_images) {
- if (result_images == nullptr) {
- return;
- }
-
- for (int i = 0; i < num_images; ++i) {
- if (result_images[i].data != nullptr) {
- free(result_images[i].data);
- result_images[i].data = nullptr;
- }
- }
-
- free(result_images);
+sd_status_t sd::generate_video(sd_ctx_t* ctx, const VideoConfig& config, VideoGenerationResult** result_out, const sd_opt_t* source) noexcept {
+ if (result_out)
+ *result_out = nullptr;
+ return sd::api::call([&] { return run_video(ctx, &config, result_out, source); }, SD_STATUS_GENERATION_FAILED);
}
diff --git a/src/upscaler.cpp b/src/upscaler.cpp
index 82aa1f4..9a16c0a 100644
--- a/src/upscaler.cpp
+++ b/src/upscaler.cpp
@@ -1,4 +1,6 @@
#include "upscaler.h"
+#include "api/options.h"
+#include "api/result.h"
#include "core/ggml_extend_backend.h"
#include "core/util.h"
#include "generation_info.h"
@@ -60,13 +62,12 @@ sd::Tensor<float> UpscalerGGML::upscale_tensor(const sd::Tensor<float>& input_te
return sd::pipeline::upscale_image_tensor(*model, input_tensor, n_threads, tile_size);
}
-sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor) {
+sd::Image UpscalerGGML::upscale(sd::Image input_image) {
sd::ParallelScope tensor_scope(&tensor_executor);
- // upscale_factor, unused for RealESRGAN_x4plus_anime_6B.pth
- sd_image_t upscaled_image = {0, 0, 0, nullptr};
- const int scale = get_scale();
- int output_width = (int)input_image.width * scale;
- int output_height = (int)input_image.height * scale;
+ sd::Image upscaled_image = {0, 0, 0, nullptr};
+ const int scale = get_scale();
+ int output_width = (int)input_image.width * scale;
+ int output_height = (int)input_image.height * scale;
LOG_INFO("upscaling from (%i x %i) to (%i x %i)",
input_image.width, input_image.height, output_width, output_height);
@@ -77,81 +78,83 @@ sd_image_t UpscalerGGML::upscale(sd_image_t input_image, uint32_t upscale_factor
if (upscaled.empty()) {
return upscaled_image;
}
- sd_image_t upscaled_data = tensor_to_sd_image(upscaled);
- int64_t t3 = ggml_time_ms();
+ sd::Image upscaled_data = tensor_to_sd_image(upscaled);
+ int64_t t3 = ggml_time_ms();
LOG_INFO("input_image_tensor upscaled, taking %.2fs", (t3 - t0) / 1000.0f);
upscaled_image = upscaled_data;
return upscaled_image;
}
-struct upscaler_ctx_t {
- UpscalerGGML* upscaler = nullptr;
+struct sd_upscaler {
+ std::unique_ptr<UpscalerGGML> upscaler;
};
-upscaler_ctx_t* new_upscaler_ctx(const char* esrgan_path_c_str,
- bool direct,
- int n_threads,
- int tile_size,
- const char* backend,
- const char* params_backend) {
- upscaler_ctx_t* upscaler_ctx = (upscaler_ctx_t*)malloc(sizeof(upscaler_ctx_t));
- if (upscaler_ctx == nullptr) {
- return nullptr;
- }
- std::string esrgan_path(esrgan_path_c_str);
-
- upscaler_ctx->upscaler = new UpscalerGGML(n_threads, direct, tile_size, SAFE_STR(backend), SAFE_STR(params_backend));
- if (upscaler_ctx->upscaler == nullptr) {
- return nullptr;
- }
-
- if (!upscaler_ctx->upscaler->load_from_file(esrgan_path, n_threads)) {
- delete upscaler_ctx->upscaler;
- upscaler_ctx->upscaler = nullptr;
- free(upscaler_ctx);
- return nullptr;
- }
- return upscaler_ctx;
+sd_status_t sd_upscaler_create(const sd_upscaler_opt_t* opt, sd_upscaler_t** out) {
+ if (out)
+ *out = nullptr;
+ return sd::api::call([&] {
+ if (!out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(opt, sd::api::Upscaler, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ auto result = std::make_unique<sd_upscaler_t>();
+ const auto& ctx = config.context;
+ result->upscaler = std::make_unique<UpscalerGGML>(ctx.n_threads, config.post.conv_direct, config.post.tile_size,
+ SAFE_STR(ctx.backend), SAFE_STR(ctx.params_backend));
+ if (!result->upscaler->load_from_file(ctx.model_path, ctx.n_threads))
+ return sd::api::error(SD_STATUS_LOAD_FAILED, "model_path", "could not load upscaler");
+ *out = result.release();
+ return SD_STATUS_OK;
+ },
+ SD_STATUS_LOAD_FAILED);
}
-bool upscale(upscaler_ctx_t* upscaler_ctx,
- sd_image_t input_image,
- uint32_t upscale_factor,
- sd_image_generation_result_t** result_out) try {
- if (result_out)
- *result_out = nullptr;
- if (!result_out)
- return false;
- if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) {
- return false;
- }
-
- sd_image_t* result_images = (sd_image_t*)calloc(1, sizeof(sd_image_t));
- if (result_images == nullptr) {
- return false;
- }
-
- auto release_images = [](sd_image_t* images) { free_sd_images(images, 1); };
- std::unique_ptr<sd_image_t, decltype(release_images)> image_owner(result_images, release_images);
- const std::vector<nlohmann::json> infos{sd::upscale_generation_info(upscale_factor)};
- result_images[0] = upscaler_ctx->upscaler->upscale(input_image, upscale_factor);
- if (!result_images[0].data)
- return false;
+void sd_upscaler_free(sd_upscaler_t* upscaler) {
+ delete upscaler;
+}
- return sd::make_image_generation_result(image_owner.release(), 1, infos, result_out);
-} catch (const std::exception& e) {
- LOG_ERROR("upscale failed: %s", e.what());
- return false;
+sd_status_t sd_upscale(sd_upscaler_t* upscaler, const sd_upscale_opt_t* opt, sd_result_t** out) {
+ if (out)
+ *out = nullptr;
+ return sd::api::call([&] {
+ if (!out)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "out is required");
+ sd::OptConfig config;
+ auto status = sd::api::resolve(opt, sd::api::Upscale, config);
+ if (status != SD_STATUS_OK)
+ return status;
+ if (!upscaler || !upscaler->upscaler)
+ return sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required");
+ const auto& image = config.post.image;
+ const auto scale = upscaler->upscaler->get_scale();
+ if (scale <= 0 || image.width > uint32_t(INT32_MAX / scale) || image.height > uint32_t(INT32_MAX / scale))
+ return sd::api::error(SD_STATUS_OUT_OF_RANGE, "image", "upscaled dimensions exceed the supported range");
+ auto release = [](sd::Image* images) { sd::free_images(images, 1); };
+ std::unique_ptr<sd::Image, decltype(release)> images(static_cast<sd::Image*>(calloc(1, sizeof(sd::Image))), release);
+ if (!images)
+ return sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result");
+ images.get()[0] = upscaler->upscaler->upscale(image);
+ if (!images->data)
+ return sd::api::error(SD_STATUS_GENERATION_FAILED, nullptr, "upscale failed");
+ const std::vector<nlohmann::json> infos{sd::upscale_generation_info(scale)};
+ return sd::make_image_result(images.release(), 1, infos, out) ? SD_STATUS_OK
+ : sd::api::error(SD_STATUS_OUT_OF_MEMORY, nullptr, "could not allocate result metadata");
+ },
+ SD_STATUS_GENERATION_FAILED);
}
-int get_upscale_factor(upscaler_ctx_t* upscaler_ctx) {
- if (upscaler_ctx == nullptr || upscaler_ctx->upscaler == nullptr) {
- return 1;
+int sd_upscaler_get_scale(const sd_upscaler_t* upscaler) {
+ sd::api::clear_error();
+ if (!upscaler || !upscaler->upscaler) {
+ sd::api::error(SD_STATUS_INVALID_ARGUMENT, nullptr, "upscaler is required");
+ return 0;
}
- return upscaler_ctx->upscaler->get_scale();
+ return upscaler->upscaler->get_scale();
}
-int get_upscaler_model_scale(const char* model_path) {
+int sd_upscaler_probe_scale(const char* model_path) {
if (model_path == nullptr || model_path[0] == '\0') {
return 0;
}
@@ -178,13 +181,3 @@ int get_upscaler_model_scale(const char* model_path) {
return 0;
}
}
-
-void free_upscaler_ctx(upscaler_ctx_t* upscaler_ctx) {
- if (!upscaler_ctx)
- return;
- if (upscaler_ctx->upscaler != nullptr) {
- delete upscaler_ctx->upscaler;
- upscaler_ctx->upscaler = nullptr;
- }
- free(upscaler_ctx);
-}
diff --git a/src/upscaler.h b/src/upscaler.h
index f3d0938..e3cf09f 100644
--- a/src/upscaler.h
+++ b/src/upscaler.h
@@ -1,6 +1,8 @@
#ifndef __SD_UPSCALER_H__
#define __SD_UPSCALER_H__
+#include "core/media.h"
+
#include "core/ggml_extend_backend.h"
#include "core/tensor.hpp"
#include "model_manager.h"
@@ -35,7 +37,7 @@ struct UpscalerGGML {
void set_max_graph_vram_bytes(size_t max_vram_bytes);
int get_scale();
sd::Tensor<float> upscale_tensor(const sd::Tensor<float>& input_tensor);
- sd_image_t upscale(sd_image_t input_image, uint32_t upscale_factor);
+ sd::Image upscale(sd::Image input_image);
};
#endif // __SD_UPSCALER_H__