Commit 1e1fc826 for libheif
commit 1e1fc826171e19f48102dc4f396b7d31a5c2b574
Author: Dirk Farin <dirk.farin@gmail.com>
Date: Mon Oct 26 10:24:08 2026 +0100
sequences: credit decode work to the draining decoder, share the two-chunk test rewrite
Follow-up to #1939 by @dlemstra.
- Frames are now taken from the active decoder, which can have a different
compression format than the decoder of the next input sample when the sample
description changes. Credit the decode work budget (GHSA-48p4) to the format of
the decoder that returned the frame.
- Remove the TODO about collecting pending output before a decoder switch; this is
what the draining does now.
- Document that the configuration NALs are now sent at the start of every chunk,
also to a decoder shared by consecutive chunks (repeated parameter sets or
sequence headers are harmless), and that the remaining "Did not decode all
frames" branch is a guard that is not reached.
- Move the byte-level 'moov' rewrite of the new test (first sample in a chunk of
its own, optionally with a second sample entry) into tests/sequence_file_builder.h
and use it in image_description_metadata as well, replacing that test's hand-built
stsc/stco tables. The #1938 test now checks all three frames with two sample
descriptions, which the draining makes possible.
diff --git a/libheif/sequences/track_visual.cc b/libheif/sequences/track_visual.cc
index f38e77d3..ed22736c 100644
--- a/libheif/sequences/track_visual.cc
+++ b/libheif/sequences/track_visual.cc
@@ -288,10 +288,6 @@ Result<std::shared_ptr<HeifPixelImage> > Track_Visual::decode_next_image_sample(
const std::shared_ptr<Chunk>& chunk = m_chunks[chunk_idx];
- // TODO: A sample-description change can switch codecs (e.g. HEVC to JPEG 2000).
- // Before using the new decoder, collect any pending output from the previous
- // decoder in presentation order. Selecting a decoder by the next input sample
- // alone does not poll the previous decoder after the switch.
auto decoder = chunk->get_decoder();
if (!decoder) {
// Should not happen: Track_Visual::load() rejects any visual track that has a
@@ -328,7 +324,10 @@ Result<std::shared_ptr<HeifPixelImage> > Track_Visual::decode_next_image_sample(
// Credit the returned frame and one potentially non-displayed sample.
// Do this at the decoder boundary, even if later image postprocessing fails.
- decode_work_debt = (decode_work_debt > 2) ? decode_work_debt - 2 : 0;
+ // The frame comes from the active decoder, which can have a different format
+ // than the decoder of the next input sample when the sample description changes.
+ uint16_t& output_work_debt = m_decode_work_debt[m_active_decoder->get_compression_format()];
+ output_work_debt = (output_work_debt > 2) ? output_work_debt - 2 : 0;
break;
}
@@ -376,7 +375,10 @@ Result<std::shared_ptr<HeifPixelImage> > Track_Visual::decode_next_image_sample(
m_next_sample_to_be_decoded++;
// Send the decoder configuration when we send the first sample of the chunk.
- // The configuration NALs might change for each chunk.
+ // The configuration NALs might change for each chunk. A decoder that is shared by
+ // consecutive chunks with the same sample description receives them again, which is
+ // harmless (repeated parameter sets / sequence headers), and a decoder that was released
+ // after draining needs them to start again.
const bool is_first_sample = (sample_idx_in_chunk == chunk->first_sample_number());
// --- Push data into the decoder.
@@ -392,6 +394,8 @@ Result<std::shared_ptr<HeifPixelImage> > Track_Visual::decode_next_image_sample(
m_active_decoder = decoder;
}
else {
+ // Not reached: at end of input, there is always an active decoder (we pushed at least one sample)
+ // and the draining above either returns a frame or reports an error. Guard against endless looping anyway.
return Error(heif_error_Decoder_plugin_error,
heif_suberror_Unspecified,
"Did not decode all frames");
diff --git a/tests/image_description_metadata.cc b/tests/image_description_metadata.cc
index 0a6f3144..f20cd650 100644
--- a/tests/image_description_metadata.cc
+++ b/tests/image_description_metadata.cc
@@ -32,9 +32,12 @@
#include "libheif/heif_uncompressed.h"
#include "libheif/heif_sequences.h"
#include "sequences/seq_boxes.h"
+#include "sequence_file_builder.h"
#include <algorithm>
#include <cstring>
#include <memory>
+#include <optional>
+#include <utility>
// Holds the non-default values for every ImageDescription metadata field so
@@ -345,22 +348,20 @@ static std::vector<uint8_t> encode_metadata_sequence()
static void add_sequence_metadata(std::vector<uint8_t>& file, const std::vector<uint8_t>& icc,
bool multiple_descriptions = false)
{
- auto reader = std::make_shared<StreamReader_memory>(file.data(), file.size(), false);
- BitstreamRange range(reader, file.size());
- while (reader->get_position() < file.size()) {
- const size_t start = reader->get_position();
- BoxHeader header;
- REQUIRE(header.parse_header(range) == Error::Ok);
- REQUIRE(header.get_box_size() >= header.get_header_size());
- REQUIRE(header.get_box_size() <= file.size() - start);
- if (header.get_short_type() != fourcc("moov")) {
- REQUIRE(reader->seek(start + header.get_box_size()));
- continue;
+ REQUIRE(seqfile::rewrite_moov_keeping_offsets(file, [&](const std::vector<uint8_t>& encoded_moov) {
+ // With two descriptions, the first sample uses the first sample entry and the
+ // remaining samples use a copy of it, which gets a different 'clli' below.
+ std::vector<uint8_t> moov_data = encoded_moov;
+ if (multiple_descriptions) {
+ auto split = seqfile::split_first_sample_into_own_chunk(encoded_moov, true);
+ REQUIRE(split);
+ moov_data = std::move(*split);
}
- REQUIRE(reader->seek(start));
- BitstreamRange movie_range(reader, header.get_box_size());
+
+ auto reader = std::make_shared<StreamReader_memory>(moov_data.data(), moov_data.size(), false);
+ BitstreamRange range(reader, moov_data.size());
std::shared_ptr<Box> moov;
- REQUIRE(Box::read(movie_range, &moov, heif_get_global_security_limits()) == Error::Ok);
+ REQUIRE(Box::read(range, &moov, heif_get_global_security_limits()) == Error::Ok);
auto trak = moov->get_child_box<Box_trak>();
REQUIRE(trak != nullptr);
auto mdia = trak->get_child_box<Box_mdia>();
@@ -371,93 +372,39 @@ static void add_sequence_metadata(std::vector<uint8_t>& file, const std::vector<
REQUIRE(stbl != nullptr);
auto stsd = stbl->get_child_box<Box_stsd>();
REQUIRE(stsd != nullptr);
- auto entry = std::const_pointer_cast<Box>(stsd->get_sample_entry(0));
- REQUIRE(entry != nullptr);
- nclx_profile profile;
- profile.set_colour_primaries(9);
- profile.set_transfer_characteristics(16);
- profile.set_matrix_coefficients(9);
- profile.set_full_range_flag(false);
- auto colr = std::make_shared<Box_colr>();
- colr->set_color_profile(std::make_shared<color_profile_nclx>(profile));
- entry->replace_child_box(colr);
- auto raw_colr = std::make_shared<Box_colr>();
- raw_colr->set_color_profile(std::make_shared<color_profile_raw>(fourcc("prof"), icc));
- entry->append_child_box(raw_colr);
- auto clli = std::make_shared<Box_clli>();
- clli->clli = {1000, 400};
- entry->append_child_box(clli);
- auto mdcv = std::make_shared<Box_mdcv>();
- mdcv->mdcv.max_display_mastering_luminance = 10000000;
- mdcv->mdcv.min_display_mastering_luminance = 50;
- entry->append_child_box(mdcv);
- auto pasp = std::make_shared<Box_pasp>();
- pasp->hSpacing = 4;
- pasp->vSpacing = 3;
- entry->append_child_box(pasp);
- if (multiple_descriptions) {
- StreamWriter entry_writer;
- REQUIRE(entry->write(entry_writer) == Error::Ok);
- const auto entry_data = entry_writer.get_data();
- auto entry_reader = std::make_shared<StreamReader_memory>(entry_data.data(), entry_data.size(), false);
- BitstreamRange entry_range(entry_reader, entry_data.size());
- std::shared_ptr<Box> second_entry;
- REQUIRE(Box::read(entry_range, &second_entry, heif_get_global_security_limits()) == Error::Ok);
- second_entry->get_child_box<Box_clli>()->clli = {2000, 800};
- stsd->add_sample_entry(second_entry);
-
- auto stco = stbl->get_child_box<Box_stco>();
- auto stsz = stbl->get_child_box<Box_stsz>();
- REQUIRE(stco != nullptr);
- REQUIRE(stsz != nullptr);
- REQUIRE(stco->get_offsets().size() == 1);
- REQUIRE(stsz->num_samples() == 3);
- StreamWriter tables;
- tables.write32(40);
- tables.write32(fourcc("stsc"));
- tables.write32(0);
- tables.write32(2);
- for (uint32_t value : {1u, 1u, 1u, 2u, 2u, 2u}) {
- tables.write32(value);
- }
- tables.write32(24);
- tables.write32(fourcc("stco"));
- tables.write32(0);
- tables.write32(2);
- tables.write32(stco->get_offsets()[0]);
- const uint32_t first_sample_size = stsz->has_fixed_sample_size() ?
- stsz->get_fixed_sample_size() : stsz->get_sample_sizes()[0];
- tables.write32(stco->get_offsets()[0] + first_sample_size);
- const auto table_data = tables.get_data();
- auto table_reader = std::make_shared<StreamReader_memory>(table_data.data(), table_data.size(), false);
- BitstreamRange table_range(table_reader, table_data.size());
- for (int i = 0; i < 2; ++i) {
- std::shared_ptr<Box> table;
- REQUIRE(Box::read(table_range, &table, heif_get_global_security_limits()) == Error::Ok);
- if (i == 0) {
- auto mapping = std::dynamic_pointer_cast<Box_stsc>(table);
- REQUIRE(mapping != nullptr);
- REQUIRE(stbl->replace_child_box(mapping));
- }
- else {
- auto offsets = std::dynamic_pointer_cast<Box_stco>(table);
- REQUIRE(offsets != nullptr);
- REQUIRE(stbl->replace_child_box(offsets));
- }
- }
+ REQUIRE(stsd->get_num_sample_entries() == (multiple_descriptions ? 2 : 1));
+
+ for (size_t i = 0; i < stsd->get_num_sample_entries(); i++) {
+ auto entry = std::const_pointer_cast<Box>(stsd->get_sample_entry(i));
+ REQUIRE(entry != nullptr);
+ nclx_profile profile;
+ profile.set_colour_primaries(9);
+ profile.set_transfer_characteristics(16);
+ profile.set_matrix_coefficients(9);
+ profile.set_full_range_flag(false);
+ auto colr = std::make_shared<Box_colr>();
+ colr->set_color_profile(std::make_shared<color_profile_nclx>(profile));
+ entry->replace_child_box(colr);
+ auto raw_colr = std::make_shared<Box_colr>();
+ raw_colr->set_color_profile(std::make_shared<color_profile_raw>(fourcc("prof"), icc));
+ entry->append_child_box(raw_colr);
+ auto clli = std::make_shared<Box_clli>();
+ clli->clli = (i == 0) ? heif_content_light_level{1000, 400} : heif_content_light_level{2000, 800};
+ entry->append_child_box(clli);
+ auto mdcv = std::make_shared<Box_mdcv>();
+ mdcv->mdcv.max_display_mastering_luminance = 10000000;
+ mdcv->mdcv.min_display_mastering_luminance = 50;
+ entry->append_child_box(mdcv);
+ auto pasp = std::make_shared<Box_pasp>();
+ pasp->hSpacing = 4;
+ pasp->vSpacing = 3;
+ entry->append_child_box(pasp);
}
+
StreamWriter output;
REQUIRE(moov->write(output) == Error::Ok);
- const auto data = output.get_data();
- // Leave a same-sized free box in place and append the modified movie box,
- // preserving the offsets of all encoded samples.
- std::memcpy(file.data() + start + 4, "free", 4);
- std::fill(file.begin() + start + header.get_header_size(),
- file.begin() + start + header.get_box_size(), 0);
- file.insert(file.end(), data.begin(), data.end());
- return;
- }
- FAIL("Encoded sequence has no movie box");
+ return std::optional<std::vector<uint8_t>>(output.get_data());
+ }));
}
TEST_CASE("sequence sample-entry colour metadata is applied to every decoded frame")
@@ -491,10 +438,7 @@ TEST_CASE("sequence sample-entry colour metadata is applied to every decoded fra
options->ignore_sequence_editlist = 1;
options->output_image_nclx_profile_passthrough = 1;
options->autocorrect_broken_input = autocorrect;
- // Check both descriptions across the chunk boundary. Final flushing with
- // multiple decoder instances is not yet supported by the sequence decoder.
- const int frames_to_check = with_metadata && multiple_descriptions ? 2 : 3;
- for (int frame = 0; frame < frames_to_check; ++frame) {
+ for (int frame = 0; frame < 3; ++frame) {
INFO("frame: " << frame);
heif_image* image = nullptr;
REQUIRE(heif_track_decode_next_image(track, &image,
@@ -548,7 +492,7 @@ TEST_CASE("sequence sample-entry colour metadata is applied to every decoded fra
}
heif_image_release(image);
}
- if (!(with_metadata && multiple_descriptions)) {
+ {
heif_image* image = nullptr;
CHECK(heif_track_decode_next_image(track, &image, heif_colorspace_undefined,
heif_chroma_undefined, options).code == heif_error_End_of_sequence);
diff --git a/tests/sequence_file_builder.h b/tests/sequence_file_builder.h
index 3a0f5344..d6ffc441 100644
--- a/tests/sequence_file_builder.h
+++ b/tests/sequence_file_builder.h
@@ -31,9 +31,13 @@
#ifndef LIBHEIF_TESTS_SEQUENCE_FILE_BUILDER_H
#define LIBHEIF_TESTS_SEQUENCE_FILE_BUILDER_H
+#include <algorithm>
#include <cstdint>
#include <initializer_list>
#include <limits>
+#include <optional>
+#include <string>
+#include <utility>
#include <vector>
namespace seqfile {
@@ -364,6 +368,162 @@ inline std::vector<uint8_t> build_sequence_file(const SequenceFileParams& p)
return concat({ftyp, moov, mdat});
}
+
+// --- Rewriting the 'moov' box of an encoded sequence
+
+inline uint32_t get32(const std::vector<uint8_t>& v, size_t offset)
+{
+ return (static_cast<uint32_t>(v[offset]) << 24) |
+ (static_cast<uint32_t>(v[offset + 1]) << 16) |
+ (static_cast<uint32_t>(v[offset + 2]) << 8) |
+ static_cast<uint32_t>(v[offset + 3]);
+}
+
+// Replace the top-level 'moov' box by a 'free' box of the same size and append the 'moov' box returned by
+// 'rewrite' (std::optional<std::vector<uint8_t>>(const std::vector<uint8_t>& moov)) at the end of the file.
+// This keeps the chunk offsets into 'mdat' valid. Returns false if there is no well-formed top-level 'moov'
+// box or if 'rewrite' returns no box.
+template <typename Rewrite>
+inline bool rewrite_moov_keeping_offsets(std::vector<uint8_t>& file, Rewrite rewrite)
+{
+ for (size_t offset = 0; file.size() - offset >= 8;) {
+ const uint32_t size = get32(file, offset);
+ if (size < 8 || size > file.size() - offset) {
+ return false;
+ }
+
+ if (std::string(file.begin() + offset + 4, file.begin() + offset + 8) == "moov") {
+ const std::vector<uint8_t> moov(file.begin() + offset, file.begin() + offset + size);
+ std::optional<std::vector<uint8_t>> replacement = rewrite(moov);
+ if (!replacement) {
+ return false;
+ }
+
+ std::copy_n("free", 4, file.begin() + offset + 4);
+ std::fill(file.begin() + offset + 8, file.begin() + offset + size, 0);
+ append(file, *replacement);
+ return true;
+ }
+
+ offset += size;
+ }
+
+ return false;
+}
+
+namespace detail {
+
+inline bool split_first_sample_into_own_chunk(const std::vector<uint8_t>& data, bool different_descriptions,
+ uint32_t& num_samples, uint32_t& first_sample_size,
+ std::vector<uint8_t>& output)
+{
+ for (size_t offset = 0; offset < data.size();) {
+ if (data.size() - offset < 8) {
+ return false;
+ }
+
+ const uint32_t size = get32(data, offset);
+ if (size < 8 || size > data.size() - offset) {
+ return false;
+ }
+
+ const std::string type(data.begin() + offset + 4, data.begin() + offset + 8);
+ std::vector<uint8_t> payload(data.begin() + offset + 8, data.begin() + offset + size);
+
+ if (type == "moov" || type == "trak" || type == "mdia" || type == "minf" || type == "stbl") {
+ // Read stsz before rewriting stco, regardless of their order in the file.
+ if (type == "stbl") {
+ for (size_t child = 0; payload.size() - child >= 8;) {
+ const uint32_t child_size = get32(payload, child);
+ if (child_size < 8 || child_size > payload.size() - child) {
+ return false;
+ }
+
+ if (std::string(payload.begin() + child + 4, payload.begin() + child + 8) == "stsz") {
+ if (child_size < 20) {
+ return false;
+ }
+
+ num_samples = get32(payload, child + 16);
+ first_sample_size = get32(payload, child + 12);
+ if (first_sample_size == 0) {
+ if (child_size < 24) {
+ return false;
+ }
+ first_sample_size = get32(payload, child + 20);
+ }
+ }
+
+ child += child_size;
+ }
+ }
+
+ std::vector<uint8_t> rewritten;
+ if (!split_first_sample_into_own_chunk(payload, different_descriptions, num_samples, first_sample_size,
+ rewritten)) {
+ return false;
+ }
+ payload = std::move(rewritten);
+ }
+ else if (type == "stsd" && different_descriptions) {
+ if (payload.size() < 8 || get32(payload, 4) != 1) {
+ return false;
+ }
+
+ const std::vector<uint8_t> entry(payload.begin() + 8, payload.end());
+ payload[7] = 2;
+ append(payload, entry);
+ }
+ else if (type == "stsc") {
+ if (num_samples < 2) {
+ return false;
+ }
+
+ payload.clear();
+ for (uint32_t value : {0u, 2u,
+ 1u, 1u, 1u,
+ 2u, num_samples - 1, different_descriptions ? 2u : 1u}) {
+ put32(payload, value);
+ }
+ }
+ else if (type == "stco") {
+ if (payload.size() < 12 || get32(payload, 4) != 1 || first_sample_size == 0) {
+ return false;
+ }
+
+ const uint32_t first_offset = get32(payload, 8);
+ payload.clear();
+ for (uint32_t value : {0u, 2u, first_offset, first_offset + first_sample_size}) {
+ put32(payload, value);
+ }
+ }
+
+ append(output, box(type.c_str(), payload));
+ offset += size;
+ }
+
+ return true;
+}
+
+} // namespace detail
+
+// Rewrite the 'moov' box of a single-track sequence that stores all samples in one chunk ('stco' with 32-bit
+// offsets), so that the first sample is in a chunk of its own and the remaining samples in a second chunk.
+// With 'different_descriptions', 'stsd' gets a copy of its sample entry and the second chunk uses it.
+inline std::optional<std::vector<uint8_t>> split_first_sample_into_own_chunk(const std::vector<uint8_t>& moov,
+ bool different_descriptions)
+{
+ uint32_t num_samples = 0;
+ uint32_t first_sample_size = 0;
+ std::vector<uint8_t> output;
+ if (!detail::split_first_sample_into_own_chunk(moov, different_descriptions, num_samples, first_sample_size,
+ output)) {
+ return std::nullopt;
+ }
+
+ return output;
+}
+
} // namespace seqfile
#endif
diff --git a/tests/sequence_vfr_durations.cc b/tests/sequence_vfr_durations.cc
index d8e30e8d..325618d2 100644
--- a/tests/sequence_vfr_durations.cc
+++ b/tests/sequence_vfr_durations.cc
@@ -184,92 +184,13 @@ std::vector<uint32_t> decode_durations(const std::vector<uint8_t>& file,
} // namespace
-static uint32_t sequence_box_u32(const std::vector<uint8_t>& data, size_t offset)
-{
- REQUIRE(offset + 4 <= data.size());
- return (static_cast<uint32_t>(data[offset]) << 24) |
- (static_cast<uint32_t>(data[offset + 1]) << 16) |
- (static_cast<uint32_t>(data[offset + 2]) << 8) | data[offset + 3];
-}
-
-static std::vector<uint8_t> split_sequence_chunks(const std::vector<uint8_t>& data,
- bool different_descriptions,
- uint32_t& first_sample_size)
-{
- std::vector<uint8_t> output;
- for (size_t offset = 0; offset < data.size();) {
- const uint32_t size = sequence_box_u32(data, offset);
- REQUIRE(size >= 8);
- REQUIRE(size <= data.size() - offset);
- const std::string type(data.begin() + offset + 4, data.begin() + offset + 8);
- std::vector<uint8_t> payload(data.begin() + offset + 8, data.begin() + offset + size);
- if (type == "moov" || type == "trak" || type == "mdia" || type == "minf" || type == "stbl") {
- // Read stsz before rewriting stco, regardless of their order in the file.
- if (type == "stbl") {
- for (size_t child = 0; child < payload.size();) {
- const uint32_t child_size = sequence_box_u32(payload, child);
- REQUIRE(child_size >= 8);
- REQUIRE(child_size <= payload.size() - child);
- if (std::string(payload.begin() + child + 4, payload.begin() + child + 8) == "stsz") {
- REQUIRE(sequence_box_u32(payload, child + 16) == 3);
- first_sample_size = sequence_box_u32(payload, child + 12);
- if (first_sample_size == 0) {
- first_sample_size = sequence_box_u32(payload, child + 20);
- }
- }
- child += child_size;
- }
- }
- payload = split_sequence_chunks(payload, different_descriptions, first_sample_size);
- }
- else if (type == "stsd" && different_descriptions) {
- REQUIRE(sequence_box_u32(payload, 4) == 1);
- const std::vector<uint8_t> entry(payload.begin() + 8, payload.end());
- payload[7] = 2;
- seqfile::append(payload, entry);
- }
- else if (type == "stsc") {
- payload.clear();
- for (uint32_t value : {0u, 2u, 1u, 1u, 1u, 2u, 2u, different_descriptions ? 2u : 1u}) {
- seqfile::put32(payload, value);
- }
- }
- else if (type == "stco") {
- REQUIRE(sequence_box_u32(payload, 4) == 1);
- REQUIRE(first_sample_size > 0);
- const uint32_t first_offset = sequence_box_u32(payload, 8);
- payload.clear();
- for (uint32_t value : {0u, 2u, first_offset, first_offset + first_sample_size}) {
- seqfile::put32(payload, value);
- }
- }
- seqfile::append(output, seqfile::box(type.c_str(), payload));
- offset += size;
- }
- return output;
-}
-
static std::vector<uint8_t> make_two_chunk_sequence(std::vector<uint8_t> file,
bool different_descriptions)
{
- for (size_t offset = 0; offset < file.size();) {
- const uint32_t size = sequence_box_u32(file, offset);
- REQUIRE(size >= 8);
- REQUIRE(size <= file.size() - offset);
- if (std::string(file.begin() + offset + 4, file.begin() + offset + 8) == "moov") {
- const std::vector<uint8_t> movie(file.begin() + offset, file.begin() + offset + size);
- uint32_t first_sample_size = 0;
- const auto replacement = split_sequence_chunks(movie, different_descriptions, first_sample_size);
- // Keep media offsets unchanged by replacing the old movie with a free box.
- std::copy_n("free", 4, file.begin() + offset + 4);
- std::fill(file.begin() + offset + 8, file.begin() + offset + size, 0);
- seqfile::append(file, replacement);
- return file;
- }
- offset += size;
- }
- FAIL("Encoded sequence has no movie box");
- return {};
+ REQUIRE(seqfile::rewrite_moov_keeping_offsets(file, [&](const std::vector<uint8_t>& moov) {
+ return seqfile::split_first_sample_into_own_chunk(moov, different_descriptions);
+ }));
+ return file;
}