Commit b96cccdb for libheif
commit b96cccdb6cbb8299f3aa3f9920de62d7863dfd9d
Author: Dirk Farin <dirk.farin@gmail.com>
Date: Mon Oct 26 09:57:22 2026 +0100
sequences: share colr/bitstream NCLX consolidation with still images, attach all sample-entry metadata
Follow-up to #1938 by @dlemstra.
- Move the colr vs. bitstream NCLX consolidation out of ImageItem::decode_image()
into consolidate_colr_and_bitstream_nclx() (nclx.cc) and use it for sequence
frames too. Sequences previously got only the full-range autocorrection, not the
warning about a CICP or range mismatch between colr and the bitstream.
- Besides colr and clli, also attach mdcv, amve, ndwt and pasp from the sample
entry, matching what still images take from their item properties.
- Collect the sample-entry metadata once per sample description in
Track_Visual::load() instead of scanning the child boxes for every frame.
- Test: replace the boolean pair with a three-state mode (no metadata, one or two
sample descriptions) to drop four duplicate combinations, add mdcv and pasp
to the sample entry and check the mismatch warnings.
diff --git a/libheif/image-items/image_item.cc b/libheif/image-items/image_item.cc
index 371dcc26..97bc8040 100644
--- a/libheif/image-items/image_item.cc
+++ b/libheif/image-items/image_item.cc
@@ -1563,65 +1563,11 @@ Result<std::shared_ptr<HeifPixelImage>> ImageItem::decode_image(const heif_decod
const auto heif_nclx = get_color_profile_nclx();
if (heif_nclx.is_defined()) {
- // Since we have a HEIF colr box, we overwrite the bitstream's CICP parameter
- // with that parameter from the colr box.
- nclx_profile consolidated_nclx = heif_nclx;
-
- // If the decoder plugin populated an NCLX profile from the bitstream's
- // color signalling (e.g. HEVC SPS VUI, AV1 sequence header), compare it
- // against the colr box. Per ISO/IEC 14496-12 and ISO/IEC 23000-22 (MIAF)
- // the colr box overrides the bitstream, but a mismatch is a strong
- // indication of a muxer bug.
- const auto bitstream_nclx = img->get_color_profile_nclx();
- if (bitstream_nclx.is_defined()) {
-
- // Check whether there is a CICP mismatch between the HEIF colr box and the compressed bitstream
- // If yes, output a warning.
-
- auto cicp_mismatch = [](uint16_t bs, uint16_t cr) {
- return bs != 2 /*unspecified*/ && cr != 2 && bs != cr;
- };
-
- if (cicp_mismatch(bitstream_nclx.m_colour_primaries, heif_nclx.m_colour_primaries) ||
- cicp_mismatch(bitstream_nclx.m_transfer_characteristics, heif_nclx.m_transfer_characteristics) ||
- cicp_mismatch(bitstream_nclx.m_matrix_coefficients, heif_nclx.m_matrix_coefficients) ||
- bitstream_nclx.m_full_range_flag != heif_nclx.m_full_range_flag) {
- std::stringstream msg;
- msg << "colr box NCLX ("
- << heif_nclx.m_colour_primaries << "/"
- << heif_nclx.m_transfer_characteristics << "/"
- << heif_nclx.m_matrix_coefficients << "/"
- << (heif_nclx.m_full_range_flag ? "full" : "limited")
- << ") disagrees with bitstream signalling ("
- << bitstream_nclx.m_colour_primaries << "/"
- << bitstream_nclx.m_transfer_characteristics << "/"
- << bitstream_nclx.m_matrix_coefficients << "/"
- << (bitstream_nclx.m_full_range_flag ? "full" : "limited")
- << "); colr takes precedence per ISO/IEC 14496-12 and ISO/IEC 23000-22 (MIAF)";
- add_decoding_warning({heif_error_Invalid_input,
- heif_suberror_NCLX_colr_VUI_mismatch,
- msg.str()});
- }
-
- // Fix full-range flag in images that are probably broken.
-
- if (options.version >= 9 && options.autocorrect_broken_input) {
-
- // Some Sony cameras mis-tag full_range_flag=0 in colr while the bitstream VUI is correct (full_range_flag=1), see issue #1770.
- // Rationale for the fix: if the bitstream explicitly says full_range=1, it probably does with for a reason.
- // Thus, we keep the full-range flag.
-
- if (bitstream_nclx.get_full_range_flag() == true &&
- heif_nclx.get_full_range_flag() == false) {
- add_decoding_warning({
- heif_error_Invalid_input,
- heif_suberror_NCLX_colr_VUI_mismatch,
- "Autocorrecting full-range flag to ON (colr=limited, bitstream=full)"
- });
-
- consolidated_nclx.set_full_range_flag(true);
- }
- }
+ std::vector<Error> warnings;
+ nclx_profile consolidated_nclx = consolidate_colr_and_bitstream_nclx(heif_nclx, img->get_color_profile_nclx(),
+ options, warnings);
+ for (auto& warning : warnings) {
+ add_decoding_warning(std::move(warning));
}
img->set_color_profile_nclx(consolidated_nclx);
diff --git a/libheif/nclx.cc b/libheif/nclx.cc
index 796d3152..ea936d2f 100644
--- a/libheif/nclx.cc
+++ b/libheif/nclx.cc
@@ -21,9 +21,11 @@
#include "nclx.h"
#include "libheif/heif_experimental.h"
+#include "libheif/heif_decoding.h"
#include <cassert>
#include <memory>
+#include <sstream>
#include <string>
#include <vector>
@@ -508,3 +510,71 @@ Error Box_colr::write(StreamWriter& writer) const
}
+
+
+nclx_profile consolidate_colr_and_bitstream_nclx(const nclx_profile& colr_nclx,
+ const nclx_profile& bitstream_nclx,
+ const heif_decoding_options& options,
+ std::vector<Error>& warnings)
+{
+ // Since we have a colr box, we overwrite the bitstream's CICP parameter
+ // with that parameter from the colr box.
+ nclx_profile consolidated_nclx = colr_nclx;
+
+ // If the decoder plugin populated an NCLX profile from the bitstream's
+ // color signalling (e.g. HEVC SPS VUI, AV1 sequence header), compare it
+ // against the colr box. Per ISO/IEC 14496-12 and ISO/IEC 23000-22 (MIAF)
+ // the colr box overrides the bitstream, but a mismatch is a strong
+ // indication of a muxer bug.
+ if (!bitstream_nclx.is_defined()) {
+ return consolidated_nclx;
+ }
+
+ // Check whether there is a CICP mismatch between the colr box and the compressed bitstream
+ // If yes, output a warning.
+
+ auto cicp_mismatch = [](uint16_t bs, uint16_t cr) {
+ return bs != 2 /*unspecified*/ && cr != 2 && bs != cr;
+ };
+
+ if (cicp_mismatch(bitstream_nclx.m_colour_primaries, colr_nclx.m_colour_primaries) ||
+ cicp_mismatch(bitstream_nclx.m_transfer_characteristics, colr_nclx.m_transfer_characteristics) ||
+ cicp_mismatch(bitstream_nclx.m_matrix_coefficients, colr_nclx.m_matrix_coefficients) ||
+ bitstream_nclx.m_full_range_flag != colr_nclx.m_full_range_flag) {
+ std::stringstream msg;
+ msg << "colr box NCLX ("
+ << colr_nclx.m_colour_primaries << "/"
+ << colr_nclx.m_transfer_characteristics << "/"
+ << colr_nclx.m_matrix_coefficients << "/"
+ << (colr_nclx.m_full_range_flag ? "full" : "limited")
+ << ") disagrees with bitstream signalling ("
+ << bitstream_nclx.m_colour_primaries << "/"
+ << bitstream_nclx.m_transfer_characteristics << "/"
+ << bitstream_nclx.m_matrix_coefficients << "/"
+ << (bitstream_nclx.m_full_range_flag ? "full" : "limited")
+ << "); colr takes precedence per ISO/IEC 14496-12 and ISO/IEC 23000-22 (MIAF)";
+ warnings.emplace_back(heif_error_Invalid_input,
+ heif_suberror_NCLX_colr_VUI_mismatch,
+ msg.str());
+ }
+
+ // Fix full-range flag in images that are probably broken.
+
+ if (options.version >= 9 && options.autocorrect_broken_input) {
+
+ // Some Sony cameras mis-tag full_range_flag=0 in colr while the bitstream VUI is correct (full_range_flag=1), see issue #1770.
+ // Rationale for the fix: if the bitstream explicitly says full_range=1, it probably does with for a reason.
+ // Thus, we keep the full-range flag.
+
+ if (bitstream_nclx.get_full_range_flag() == true &&
+ colr_nclx.get_full_range_flag() == false) {
+ warnings.emplace_back(heif_error_Invalid_input,
+ heif_suberror_NCLX_colr_VUI_mismatch,
+ "Autocorrecting full-range flag to ON (colr=limited, bitstream=full)");
+
+ consolidated_nclx.set_full_range_flag(true);
+ }
+ }
+
+ return consolidated_nclx;
+}
diff --git a/libheif/nclx.h b/libheif/nclx.h
index 1eac63b8..451f846d 100644
--- a/libheif/nclx.h
+++ b/libheif/nclx.h
@@ -225,4 +225,14 @@ private:
};
+// Combine the NCLX profile of a 'colr' box (image item property or sample entry) with the
+// profile signalled in the coded bitstream (e.g. HEVC SPS VUI, AV1 sequence header).
+// The 'colr' box takes precedence, but mismatches are reported in 'warnings' and, if requested
+// by 'options', a broken full-range flag is corrected.
+nclx_profile consolidate_colr_and_bitstream_nclx(const nclx_profile& colr_nclx,
+ const nclx_profile& bitstream_nclx,
+ const struct heif_decoding_options& options,
+ std::vector<Error>& warnings);
+
+
#endif //LIBHEIF_NCLX_H
diff --git a/libheif/sequences/track_visual.cc b/libheif/sequences/track_visual.cc
index bd66b5a6..a41fe892 100644
--- a/libheif/sequences/track_visual.cc
+++ b/libheif/sequences/track_visual.cc
@@ -101,6 +101,38 @@ Error Track_Visual::load(const std::shared_ptr<Box_trak>& trak)
}
}
+ // Collect the colour profiles and HDR metadata of each sample entry, so that we can attach them to the decoded frames.
+ // As for image item properties, a later 'colr' box replaces an earlier one of the same kind.
+
+ for (size_t i = 0; i < m_stsd->get_num_sample_entries(); i++) {
+ auto sample_entry = m_stsd->get_sample_entry(i);
+ ImageDescription& metadata = m_sample_description_metadata.emplace_back();
+
+ for (const auto& colr : sample_entry->get_child_boxes<Box_colr>()) {
+ metadata.set_color_profile(colr->get_color_profile());
+ }
+
+ if (auto clli = sample_entry->get_child_box<Box_clli>()) {
+ metadata.set_clli(clli->clli);
+ }
+
+ if (auto mdcv = sample_entry->get_child_box<Box_mdcv>()) {
+ metadata.set_mdcv(mdcv->mdcv);
+ }
+
+ if (auto amve = sample_entry->get_child_box<Box_amve>()) {
+ metadata.set_amve(amve->amve);
+ }
+
+ if (auto ndwt = sample_entry->get_child_box<Box_ndwt>()) {
+ metadata.set_nominal_diffuse_white_luminance(ndwt->get_diffuse_white_luminance());
+ }
+
+ if (auto pasp = sample_entry->get_child_box<Box_pasp>()) {
+ metadata.set_pixel_ratio(pasp->hSpacing, pasp->vSpacing);
+ }
+ }
+
const std::vector<uint32_t>& chunk_offsets = m_stco->get_offsets();
// Find sequence resolution
@@ -367,32 +399,42 @@ Result<std::shared_ptr<HeifPixelImage> > Track_Visual::decode_next_image_sample(
const SampleTiming& outputTiming = m_presentation_timeline[m_next_sample_to_be_output % m_presentation_timeline.size()];
const auto& chunk = m_chunks[outputTiming.chunkIdx];
- auto sampleDescription = m_stsd->get_sample_entry(chunk->get_sample_description_index() - 1);
- const auto bitstream_nclx = image->get_color_profile_nclx();
+ // Track::load() verified that the sample description index exists.
+ const ImageDescription& metadata = m_sample_description_metadata[chunk->get_sample_description_index() - 1];
// Sample-entry colour information takes precedence over bitstream metadata.
- for (const auto& colr : sampleDescription->get_child_boxes<Box_colr>()) {
- auto profile = colr->get_color_profile();
- if (auto nclx = std::dynamic_pointer_cast<const color_profile_nclx>(profile)) {
- auto consolidated_nclx = nclx->get_nclx_color_profile();
- if (options.version >= 9 && options.autocorrect_broken_input &&
- bitstream_nclx.is_defined() && bitstream_nclx.get_full_range_flag() &&
- !consolidated_nclx.get_full_range_flag()) {
- image->add_warning({
- heif_error_Invalid_input,
- heif_suberror_NCLX_colr_VUI_mismatch,
- "Autocorrecting full-range flag to ON (colr=limited, bitstream=full)"
- });
- consolidated_nclx.set_full_range_flag(true);
- }
- image->set_color_profile_nclx(consolidated_nclx);
- }
- else {
- image->set_color_profile(profile);
- }
+ const nclx_profile sample_entry_nclx = metadata.get_color_profile_nclx();
+ if (sample_entry_nclx.is_defined()) {
+ std::vector<Error> warnings;
+ image->set_color_profile_nclx(consolidate_colr_and_bitstream_nclx(sample_entry_nclx, image->get_color_profile_nclx(),
+ options, warnings));
+ image->add_warnings(warnings);
+ }
+
+ if (metadata.has_icc_color_profile()) {
+ image->set_color_profile_icc(metadata.get_color_profile_icc());
+ }
+
+ if (metadata.has_clli()) {
+ image->set_clli(metadata.get_clli());
}
- if (auto clli = sampleDescription->get_child_box<Box_clli>()) {
- image->set_clli(clli->clli);
+
+ if (metadata.has_mdcv()) {
+ image->set_mdcv(metadata.get_mdcv());
+ }
+
+ if (metadata.has_amve()) {
+ image->set_amve(metadata.get_amve());
+ }
+
+ if (metadata.has_nominal_diffuse_white()) {
+ image->set_nominal_diffuse_white_luminance(metadata.get_nominal_diffuse_white_luminance());
+ }
+
+ if (metadata.has_nonsquare_pixel_ratio()) {
+ uint32_t h, v;
+ metadata.get_pixel_ratio(&h, &v);
+ image->set_pixel_ratio(h, v);
}
if (m_stts) {
diff --git a/libheif/sequences/track_visual.h b/libheif/sequences/track_visual.h
index 0cff7c96..88418e88 100644
--- a/libheif/sequences/track_visual.h
+++ b/libheif/sequences/track_visual.h
@@ -22,6 +22,7 @@
#define LIBHEIF_TRACK_VISUAL_H
#include "track.h"
+#include "image/image_description.h"
#include <string>
#include <memory>
#include <vector>
@@ -90,6 +91,10 @@ private:
int m_sample_duration = 0; // TODO: pass this through encoder or handle it correctly with frame reordering
+ // Colour profiles and HDR metadata of each sample entry in 'stsd', indexed by sample description index - 1.
+ // Collected once in load() and attached to every decoded frame.
+ std::vector<ImageDescription> m_sample_description_metadata;
+
// If there is an alpha-channel track associated with this color track, we reference it from here
std::shared_ptr<Track_Visual> m_aux_alpha_track;
diff --git a/tests/image_description_metadata.cc b/tests/image_description_metadata.cc
index 17010213..0a6f3144 100644
--- a/tests/image_description_metadata.cc
+++ b/tests/image_description_metadata.cc
@@ -387,6 +387,14 @@ static void add_sequence_metadata(std::vector<uint8_t>& file, const std::vector<
auto clli = std::make_shared<Box_clli>();
clli->clli = {1000, 400};
entry->append_child_box(clli);
+ auto mdcv = std::make_shared<Box_mdcv>();
+ mdcv->mdcv.max_display_mastering_luminance = 10000000;
+ mdcv->mdcv.min_display_mastering_luminance = 50;
+ entry->append_child_box(mdcv);
+ auto pasp = std::make_shared<Box_pasp>();
+ pasp->hSpacing = 4;
+ pasp->vSpacing = 3;
+ entry->append_child_box(pasp);
if (multiple_descriptions) {
StreamWriter entry_writer;
REQUIRE(entry->write(entry_writer) == Error::Ok);
@@ -458,12 +466,16 @@ TEST_CASE("sequence sample-entry colour metadata is applied to every decoded fra
!heif_have_decoder_for_format(heif_compression_AV1)) {
SKIP("No AV1 encoder/decoder pair available");
}
- const bool with_metadata = GENERATE(false, true);
+ enum class SampleEntryMetadata { none, single_description, two_descriptions };
+ const auto metadata_mode = GENERATE(SampleEntryMetadata::none,
+ SampleEntryMetadata::single_description,
+ SampleEntryMetadata::two_descriptions);
+ const bool with_metadata = metadata_mode != SampleEntryMetadata::none;
+ const bool multiple_descriptions = metadata_mode == SampleEntryMetadata::two_descriptions;
const bool convert_to_rgb = GENERATE(false, true);
const bool autocorrect = GENERATE(false, true);
- const bool multiple_descriptions = GENERATE(false, true);
- INFO("sample-entry metadata: " << with_metadata << ", RGB conversion: " << convert_to_rgb);
- INFO("autocorrect: " << autocorrect);
+ INFO("sample-entry metadata: " << with_metadata << ", two sample descriptions: " << multiple_descriptions);
+ INFO("RGB conversion: " << convert_to_rgb << ", autocorrect: " << autocorrect);
auto file = encode_metadata_sequence();
const std::vector<uint8_t> icc = {1, 2, 3, 4, 5};
if (with_metadata) {
@@ -490,10 +502,13 @@ TEST_CASE("sequence sample-entry colour metadata is applied to every decoded fra
convert_to_rgb ? heif_chroma_444 : heif_chroma_undefined,
options).code == heif_error_Ok);
REQUIRE(image != nullptr);
- if (with_metadata && autocorrect) {
- heif_error warning{};
- REQUIRE(heif_image_get_decoding_warnings(image, 0, &warning, 1) == 1);
- CHECK(warning.subcode == heif_suberror_NCLX_colr_VUI_mismatch);
+ // With sample-entry metadata, the colr box disagrees with the bitstream (always reported)
+ // and its range is limited while the bitstream says full (autocorrected on request).
+ heif_error warnings[3]{};
+ const int num_warnings = heif_image_get_decoding_warnings(image, 0, warnings, 3);
+ CHECK(num_warnings == (with_metadata ? (autocorrect ? 2 : 1) : 0));
+ for (int i = 0; i < num_warnings; i++) {
+ CHECK(warnings[i].subcode == heif_suberror_NCLX_colr_VUI_mismatch);
}
if (with_metadata && convert_to_rgb) {
size_t stride = 0;
@@ -513,6 +528,11 @@ TEST_CASE("sequence sample-entry colour metadata is applied to every decoded fra
}
CHECK(heif_image_get_raw_color_profile_size(image) == (with_metadata ? icc.size() : 0));
CHECK(heif_image_has_content_light_level(image) == (with_metadata ? 1 : 0));
+ CHECK(heif_image_has_mastering_display_colour_volume(image) == (with_metadata ? 1 : 0));
+ uint32_t pasp_h = 0, pasp_v = 0;
+ heif_image_get_pixel_aspect_ratio(image, &pasp_h, &pasp_v);
+ CHECK(pasp_h == (with_metadata ? 4 : 1));
+ CHECK(pasp_v == (with_metadata ? 3 : 1));
if (with_metadata) {
std::vector<uint8_t> actual(icc.size());
REQUIRE(heif_image_get_raw_color_profile(image, actual.data()).code == heif_error_Ok);
@@ -521,6 +541,10 @@ TEST_CASE("sequence sample-entry colour metadata is applied to every decoded fra
heif_image_get_content_light_level(image, &clli);
CHECK(clli.max_content_light_level == (multiple_descriptions && frame > 0 ? 2000 : 1000));
CHECK(clli.max_pic_average_light_level == (multiple_descriptions && frame > 0 ? 800 : 400));
+ heif_mastering_display_colour_volume mdcv{};
+ heif_image_get_mastering_display_colour_volume(image, &mdcv);
+ CHECK(mdcv.max_display_mastering_luminance == 10000000);
+ CHECK(mdcv.min_display_mastering_luminance == 50);
}
heif_image_release(image);
}