Commit 0f5dd14c for libheif
commit 0f5dd14cf663c2f1dee61ed35c91632826d430a5
Author: Dirk Farin <dirk.farin@gmail.com>
Date: Sun Oct 4 23:41:15 2026 +0200
Refuse images with sample values above their bit depth at the encoder entry
An image that is passed to the encoder comes from the application, and
nothing guaranteed that its samples are within the bit depth it declared.
The conversion operators and the encoder libraries assume that they are.
With out-of-range 10 bit samples, x264 aborts on an internal assertion
(cost >= 0 in slicetype.c), OpenJPEG fails to encode, and the other
encoders encode the image without complaint. This is a hardening that
follows from GHSA-q7mw-2fmm-5q94, where such samples reached libsharpyuv.
Check the sample range once in Encoder::convert_colorspace_for_encoding(),
which still images, grid and 'tili' tiles, alpha images, thumbnails and
sequences all pass through, and in ImageItem_uncompressed::add_image_tile(),
which bypasses it. Such an image is now rejected with a usage error that
names the plane. The sharp-yuv operator keeps its own check, because it
also runs on decoded images.
The test uncompressed_interleaved_alpha_plane filled its 10 bit planes with
the byte 0x80, i.e. with the sample value 0x8080. It now uses values that
are within the bit depth.
diff --git a/libheif/api/libheif/heif_image.h b/libheif/api/libheif/heif_image.h
index 317bd646..fe7abc50 100644
--- a/libheif/api/libheif/heif_image.h
+++ b/libheif/api/libheif/heif_image.h
@@ -373,7 +373,8 @@ heif_error heif_image_create(int width, int height,
*
* <p>Planes with a bit depth of 9 to 16 bits store each sample in a 16-bit word. The sample
* values that are written into the plane have to be within the range of {@code bit_depth} bits,
- * i.e. the unused upper bits must be zero.
+ * i.e. the unused upper bits must be zero. The encoding functions return an error for an image
+ * that contains larger values.
*
* <p>An image with an interleaved chroma format carries its alpha inside the interleaved
* plane. Adding a separate {@code heif_channel_Alpha} plane to such an image is rejected
diff --git a/libheif/codecs/encoder.cc b/libheif/codecs/encoder.cc
index 17927d7f..67e13160 100644
--- a/libheif/codecs/encoder.cc
+++ b/libheif/codecs/encoder.cc
@@ -142,6 +142,14 @@ Result<std::shared_ptr<HeifPixelImage>> Encoder::convert_colorspace_for_encoding
return err;
}
+ // The image comes from the application, so nothing guarantees that its samples are within
+ // the bit depth it declared. The conversion operators and the encoder libraries behind the
+ // plugins assume that they are (libsharpyuv indexes its gamma tables with the samples, x264
+ // aborts on an internal assertion), so this is checked once here for all of them.
+ if (Error err = image->check_sample_value_ranges()) {
+ return err;
+ }
+
heif_colorspace colorspace = image->get_colorspace();
heif_chroma chroma = image->get_chroma_format();
diff --git a/libheif/color-conversion/rgb2yuv_sharp.cc b/libheif/color-conversion/rgb2yuv_sharp.cc
index d394a3a0..9e054483 100644
--- a/libheif/color-conversion/rgb2yuv_sharp.cc
+++ b/libheif/color-conversion/rgb2yuv_sharp.cc
@@ -251,10 +251,10 @@ Op_Any_RGB_to_YCbCr_420_Sharp::convert_colorspace(
// libsharpyuv uses the sample values as indices into its gamma tables, which are sized
// for the bit depth we pass, and it does not check the range itself. A sample above
// that range makes it read far outside the table, and the value found there is used
- // as the index for a second lookup. The planes cannot guarantee the range: they are
- // filled by the application when encoding, or by a decoder that may hand through
- // whatever the bitstream contained. Refuse such an image here, right in front of the
- // call that depends on it.
+ // as the index for a second lookup. Images from the application are checked when they
+ // are passed to the encoder, but this operator also runs on decoded images, and a decoder
+ // may hand through whatever the bitstream contained. Refuse such an image here, right
+ // in front of the call that depends on it.
if (Error err = input->check_sample_value_ranges()) {
return Error{heif_error_Invalid_input,
heif_suberror_Unspecified,
diff --git a/libheif/image-items/unc_image.cc b/libheif/image-items/unc_image.cc
index f3315e15..9310cd4c 100644
--- a/libheif/image-items/unc_image.cc
+++ b/libheif/image-items/unc_image.cc
@@ -366,6 +366,12 @@ Error ImageItem_uncompressed::add_image_tile(uint32_t tile_x, uint32_t tile_y, c
// TODO: drop alpha
}
+ // Tiles are passed to the 'unci' encoder directly, without going through
+ // Encoder::convert_colorspace_for_encoding(), so the sample range is checked here.
+ if (Error err = image->check_sample_value_ranges()) {
+ return err;
+ }
+
Result<std::vector<uint8_t>> codedBitstreamResult = m_unc_encoder->encode_tile(image);
if (!codedBitstreamResult) {
return codedBitstreamResult.error();
diff --git a/tests/encode_plane_layout.cc b/tests/encode_plane_layout.cc
index 97618f97..2981de4c 100644
--- a/tests/encode_plane_layout.cc
+++ b/tests/encode_plane_layout.cc
@@ -39,6 +39,7 @@
#include <cstdint>
#include <cstring>
+#include <initializer_list>
#include <string>
namespace {
@@ -189,3 +190,78 @@ TEST_CASE("encoding refuses images with a non-canonical plane layout")
}
}
+
+
+// A plane with a bit depth of 9 to 15 bits stores its samples in 16-bit words, so an
+// application can fill it with values that exceed the bit depth it declared. The conversion
+// operators and the encoder libraries assume that the samples are in range (libsharpyuv uses
+// them as table indices, x264 aborts on an internal assertion), so the image is refused when
+// it is passed to the encoder, whatever the encoder is and whether or not it is converted first.
+TEST_CASE("encoding refuses images with sample values above their bit depth")
+{
+ heif_compression_format format = pick_encoder_format();
+ if (format == heif_compression_undefined) {
+ SKIP("no HEVC, AV1 or uncompressed encoder available");
+ }
+
+ struct Layout
+ {
+ const char* name;
+ heif_colorspace colorspace;
+ heif_chroma chroma;
+ std::initializer_list<heif_channel> channels;
+ };
+
+ const Layout layouts[] = {
+ {"YCbCr 4:2:0", heif_colorspace_YCbCr, heif_chroma_420, {heif_channel_Y, heif_channel_Cb, heif_channel_Cr}},
+ {"RGB", heif_colorspace_RGB, heif_chroma_444, {heif_channel_R, heif_channel_G, heif_channel_B}},
+ {"monochrome", heif_colorspace_monochrome, heif_chroma_monochrome, {heif_channel_Y}},
+ };
+
+ for (const Layout& layout : layouts) {
+ for (int bit_depth : {10, 12}) {
+ for (bool in_range : {true, false}) {
+ INFO(layout.name << ", " << bit_depth << " bit, in_range=" << in_range);
+
+ const uint16_t max_value = static_cast<uint16_t>((1 << bit_depth) - 1);
+ const uint16_t value = in_range ? max_value : static_cast<uint16_t>(max_value + 1);
+
+ heif_image* img = create_image(layout.colorspace, layout.chroma);
+ for (heif_channel channel : layout.channels) {
+ bool subsampled = (layout.chroma == heif_chroma_420 && channel != heif_channel_Y);
+ uint32_t w = subsampled ? W / 2 : W;
+ uint32_t h = subsampled ? H / 2 : H;
+
+ heif_error err = heif_image_add_plane(img, channel, w, h, bit_depth);
+ REQUIRE(err.code == heif_error_Ok);
+
+ size_t stride = 0;
+ uint8_t* p = heif_image_get_plane2(img, channel, &stride);
+ REQUIRE(p != nullptr);
+ for (uint32_t y = 0; y < h; y++) {
+ uint16_t* row = reinterpret_cast<uint16_t*>(p + y * stride);
+ for (uint32_t x = 0; x < w; x++) {
+ row[x] = value;
+ }
+ }
+ }
+
+ EncodeResult result = encode(img, format);
+ INFO("encode error (" << result.code << "/" << result.subcode << "): " << result.message);
+
+ const bool refused_for_range = (result.message.find("exceed its bit depth") != std::string::npos);
+ if (in_range) {
+ // Whether the encoder supports this bit depth is up to the encoder, but the image
+ // must not be refused because of its sample values.
+ CHECK(!refused_for_range);
+ }
+ else {
+ CHECK(result.code == heif_error_Usage_error);
+ CHECK(refused_for_range);
+ }
+
+ heif_image_release(img);
+ }
+ }
+ }
+}
diff --git a/tests/plane_layout.cc b/tests/plane_layout.cc
index 7a29f1ad..4656223e 100644
--- a/tests/plane_layout.cc
+++ b/tests/plane_layout.cc
@@ -264,8 +264,9 @@ TEST_CASE("convert_colorspace refuses images with a non-canonical plane layout")
// A plane stores its samples in whole bytes, so a plane with a bit depth of, say, 10 bits can
// hold larger values in its 16-bit words. HeifPixelImage::check_sample_value_ranges() is the
-// gate for that: the sharp-yuv operator uses it before it hands the samples to libsharpyuv,
-// which uses them as table indices.
+// gate for that: the encoder entry uses it for images coming from the application, and the
+// sharp-yuv operator uses it before it hands the samples to libsharpyuv, which uses them as
+// table indices.
TEST_CASE("check_sample_value_ranges")
{
auto set_sample16 = [](const std::shared_ptr<HeifPixelImage>& img, heif_channel ch, size_t idx_in_last_row, uint16_t value) {
diff --git a/tests/uncompressed_interleaved_alpha_plane.cc b/tests/uncompressed_interleaved_alpha_plane.cc
index ba373892..f3073316 100644
--- a/tests/uncompressed_interleaved_alpha_plane.cc
+++ b/tests/uncompressed_interleaved_alpha_plane.cc
@@ -77,6 +77,9 @@ const InterleavedFormat interleaved_formats[] = {
};
+// Fills every byte of the plane with 'value'. For the planes with 16-bit samples this gives
+// the sample value 0x0101 * value, which has to stay within the bit depth of the plane (the
+// encoder refuses an image with larger samples).
void fill_plane(heif_image* image, heif_channel channel, uint8_t value)
{
size_t stride = 0;
@@ -128,7 +131,7 @@ TEST_CASE("heif_image_add_plane rejects a separate alpha plane on an interleaved
err = heif_image_add_plane(image, heif_channel_interleaved, WIDTH, HEIGHT, fmt.bit_depth);
REQUIRE(err.code == heif_error_Ok);
- fill_plane(image, heif_channel_interleaved, 0x80);
+ fill_plane(image, heif_channel_interleaved, 0x02);
err = heif_image_add_plane(image, heif_channel_Alpha, WIDTH, HEIGHT, fmt.bit_depth);
CHECK(err.code == heif_error_Usage_error);