Commit 406423ccf1 for ffmpeg
commit 406423ccf117a210b3020f120cf4e6203ca18bf7
Author: YoungSoo Lee <youngsoo.lee@navercorp.com>
Date: Fri Aug 21 10:48:52 2026 +0900
lavc/hevcdec: decode tiled pictures in CTB row order
Frame threading collapses on streams split into tile columns. Decoding
follows the tile scan, walking one column from top to bottom before moving on
to the next, so the lower CTB rows are published only once the last column is
done while the following frame is already waiting for them. Adding threads
does not help.
Walk the picture in row major order instead, finishing one CTB row across all
tile columns before moving on, which matches the order rows are published in
to the order they are awaited in. Every tile column keeps its own substream
state and the decoding operations are unchanged, so the output is bit-exact.
Decoding one CTB moves into decode_ctb(), which the serial path goes through
as well, so the two traversals share a single body.
This covers pictures whose tiles are bound together in a single slice by
entry points, where the substream positions come from the slice header. The
new path is selected by the tile_interleave option: the default of -1 takes
it only with frame threading, 0 disables it and 1 takes it without frame
threading too. Both require two or more tile columns and no entropy coding
synchronization.
Signed-off-by: YoungSoo Lee <youngsoo.lee@navercorp.com>
diff --git a/doc/decoders.texi b/doc/decoders.texi
index c2aa7556ab..76b4c813dc 100644
--- a/doc/decoders.texi
+++ b/doc/decoders.texi
@@ -81,6 +81,21 @@ the corresponding elements of @option{view_ids_available}, i.e.
Same validity restrictions as for @option{view_ids_available} apply to
this option.
+@item tile_interleave
+Decode pictures split into tile columns in CTB row order rather than in tile
+scan order, so that frame threading keeps overlapping. With the tile scan the
+lower CTB rows are published only once the last tile column has been decoded,
+which stalls the frames waiting for them. Only the order in which the CTBs are
+decoded changes, the output is the same.
+
+The default of -1 takes this order for eligible pictures when frame threading is
+in use, 0 disables it, and 1 takes it for eligible pictures even without frame
+threading. A picture is eligible when it has two or more tile columns, does not
+use entropy coding synchronization and packs its tiles into slices in a layout
+the traversal supports. Every other picture is decoded in tile scan order as
+before. With slice threading, a picture whose tiles are all carried in one
+slice is still decoded tile-parallel, whatever this option is set to.
+
@end table
@section rawvideo
diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index 90c903d0c6..c413571cd2 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -2775,6 +2775,45 @@ static void hls_decode_neighbour(HEVCLocalContext *lc,
lc->ctb_up_left_flag = ((x_ctb > 0) && (y_ctb > 0) && (ctb_addr_in_slice-1 >= sps->ctb_width) && (pps->tile_id[ctb_addr_ts] == pps->tile_id[pps->ctb_addr_rs_to_ts[ctb_addr_rs-1 - sps->ctb_width]]));
}
+/* Decode one CTB from the substream in lc->sub. */
+static int decode_ctb(HEVCContext *s, const HEVCLayerContext *l,
+ HEVCLocalContext *lc, int ctb_addr_rs)
+{
+ const HEVCPPS *const pps = s->pps;
+ const HEVCSPS *const sps = pps->sps;
+ const int ctb_size = 1 << sps->log2_ctb_size;
+ const int ctb_addr_ts = pps->ctb_addr_rs_to_ts[ctb_addr_rs];
+ const int x_ctb = (ctb_addr_rs % sps->ctb_width) << sps->log2_ctb_size;
+ const int y_ctb = (ctb_addr_rs / sps->ctb_width) << sps->log2_ctb_size;
+ int ret;
+
+ hls_decode_neighbour(lc, l, pps, sps, x_ctb, y_ctb, ctb_addr_ts);
+
+ ret = ff_hevc_cabac_init(lc, pps, ctb_addr_ts);
+ if (ret < 0)
+ goto fail;
+
+ hls_sao_param(lc, l, pps, sps,
+ ctb_addr_rs % sps->ctb_width, ctb_addr_rs / sps->ctb_width);
+
+ l->deblock[ctb_addr_rs].beta_offset = lc->sh->beta_offset;
+ l->deblock[ctb_addr_rs].tc_offset = lc->sh->tc_offset;
+ l->filter_slice_edges[ctb_addr_rs] = lc->sh->slice_loop_filter_across_slices_enabled_flag;
+
+ ret = hls_coding_quadtree(lc, l, pps, sps, x_ctb, y_ctb, sps->log2_ctb_size, 0);
+ if (ret < 0)
+ goto fail;
+
+ ff_hevc_save_states(lc, pps, ctb_addr_ts + 1);
+ ff_hevc_hls_filters(lc, l, pps, x_ctb, y_ctb, ctb_size);
+
+ return ret;
+
+fail:
+ l->tab_slice_address[ctb_addr_rs] = -1;
+ return ret;
+}
+
static int hls_decode_entry(HEVCContext *s, GetBitContext *gb)
{
HEVCLocalContext *const lc = &s->local_ctx[0];
@@ -2786,7 +2825,6 @@ static int hls_decode_entry(HEVCContext *s, GetBitContext *gb)
int x_ctb = 0;
int y_ctb = 0;
int ctb_addr_ts = pps->ctb_addr_rs_to_ts[lc->sh->slice_ctb_addr_rs];
- int ret;
/* The slice segment is one buffer, so CABAC stays open across tile and
* WPP row boundaries. */
@@ -2799,31 +2837,13 @@ static int hls_decode_entry(HEVCContext *s, GetBitContext *gb)
x_ctb = (ctb_addr_rs % ((sps->width + ctb_size - 1) >> sps->log2_ctb_size)) << sps->log2_ctb_size;
y_ctb = (ctb_addr_rs / ((sps->width + ctb_size - 1) >> sps->log2_ctb_size)) << sps->log2_ctb_size;
- hls_decode_neighbour(lc, l, pps, sps, x_ctb, y_ctb, ctb_addr_ts);
- ret = ff_hevc_cabac_init(lc, pps, ctb_addr_ts);
- if (ret < 0) {
- l->tab_slice_address[ctb_addr_rs] = -1;
- return ret;
- }
-
- hls_sao_param(lc, l, pps, sps,
- x_ctb >> sps->log2_ctb_size, y_ctb >> sps->log2_ctb_size);
-
- l->deblock[ctb_addr_rs].beta_offset = lc->sh->beta_offset;
- l->deblock[ctb_addr_rs].tc_offset = lc->sh->tc_offset;
- l->filter_slice_edges[ctb_addr_rs] = lc->sh->slice_loop_filter_across_slices_enabled_flag;
-
- more_data = hls_coding_quadtree(lc, l, pps, sps, x_ctb, y_ctb, sps->log2_ctb_size, 0);
- if (more_data < 0) {
- l->tab_slice_address[ctb_addr_rs] = -1;
+ more_data = decode_ctb(s, l, lc, ctb_addr_rs);
+ if (more_data < 0)
return more_data;
- }
ctb_addr_ts++;
- ff_hevc_save_states(lc, pps, ctb_addr_ts);
- ff_hevc_hls_filters(lc, l, pps, x_ctb, y_ctb, ctb_size);
}
if (x_ctb + ctb_size >= sps->width &&
@@ -3047,6 +3067,184 @@ static int slice_substreams_init(HEVCContext *s, const H2645NAL *nal)
return 0;
}
+/* Row major traversal of tiled pictures. In tile scan order the lower CTB
+ * rows are published only after the last tile column, stalling frame
+ * threading; here each CTB row is finished across all columns first. */
+
+/* Point a substream at new data; CABAC and QP prediction restart. */
+static void substream_load(HEVCSubstream *sub, const uint8_t *data, size_t size,
+ int qp_y_seed)
+{
+ sub->data = data;
+ sub->size = size;
+ sub->opened = 0;
+
+ sub->first_qp_group = 1;
+ sub->qp_y = qp_y_seed;
+ sub->cu_qp_offset_cb = 0;
+ sub->cu_qp_offset_cr = 0;
+}
+
+/* Finish filtering the last CTB of the picture, as hls_decode_entry() does. */
+static void finish_last_ctb_filter(HEVCContext *s, const HEVCLayerContext *l,
+ HEVCLocalContext *lc, int x_ctb, int y_ctb)
+{
+ const HEVCPPS *const pps = s->pps;
+ const HEVCSPS *const sps = pps->sps;
+ const int ctb_size = 1 << sps->log2_ctb_size;
+
+ if (x_ctb + ctb_size >= sps->width && y_ctb + ctb_size >= sps->height)
+ ff_hevc_hls_filter(lc, l, pps, x_ctb, y_ctb, ctb_size);
+}
+
+static int tile_interleave_wanted(const HEVCContext *s, const HEVCPPS *pps)
+{
+ if (!s->tile_interleave || !pps || !pps->tiles_enabled_flag ||
+ pps->entropy_coding_sync_enabled_flag || pps->num_tile_columns <= 1)
+ return 0;
+
+ if (s->tile_interleave < 0 && !(s->avctx->active_thread_type & FF_THREAD_FRAME))
+ return 0;
+
+ return pps->num_tile_columns * pps->num_tile_rows > 1;
+}
+
+static int alloc_tile_streams(HEVCContext *s, unsigned n)
+{
+ if (n > s->tile_streams_alloc) {
+ void *tmp = av_realloc_array(s->tile_streams, n, sizeof(*s->tile_streams));
+
+ if (!tmp)
+ return AVERROR(ENOMEM);
+
+ s->tile_streams = tmp;
+ s->tile_streams_alloc = n;
+ }
+
+ memset(s->tile_streams, 0, n * sizeof(*s->tile_streams));
+
+ return 0;
+}
+
+static int alloc_substreams(HEVCContext *s, unsigned n)
+{
+ if (n > s->nb_substreams) {
+ HEVCSubstream *tmp = av_realloc_array(s->substreams, n, sizeof(*tmp));
+
+ if (!tmp)
+ return AVERROR(ENOMEM);
+
+ memset(tmp + s->nb_substreams, 0,
+ (n - s->nb_substreams) * sizeof(*tmp));
+
+ s->substreams = tmp;
+ s->nb_substreams = n;
+ }
+
+ return 0;
+}
+
+/* Walk the picture in raster order. A band of tile rows has one tile per
+ * column, so one substream state per column is enough; lc switches state
+ * when the column changes. */
+static int decode_tiles_row_major(HEVCContext *s, const HEVCLayerContext *l,
+ HEVCTileStream *tiles)
+{
+ const HEVCPPS *const pps = s->pps;
+ const HEVCSPS *const sps = pps->sps;
+ HEVCLocalContext *const lc = &s->local_ctx[0];
+ HEVCSubstream *cur = NULL;
+ int last_ctb_decoded = 0;
+ int ret;
+
+ for (int rs = 0; rs < sps->ctb_size; rs++) {
+ const int ts = pps->ctb_addr_rs_to_ts[rs];
+ const int tile = pps->tile_id[ts];
+ HEVCTileStream *const t = &tiles[tile];
+ HEVCSubstream *const sub = &s->substreams[tile % pps->num_tile_columns];
+
+ if (t->ended)
+ continue;
+
+ if (cur != sub) {
+ if (cur)
+ *cur = lc->sub;
+ lc->sub = *sub;
+ cur = sub;
+ }
+
+ bind_slice_ctx(lc, t->sh, t->rpl, t->collocated_ref);
+
+ if (ts == t->start_ts)
+ substream_load(&lc->sub, t->data, t->size, t->qp_y_seed);
+
+ ret = decode_ctb(s, l, lc, rs);
+ if (ret < 0)
+ return ret;
+
+ if (rs == sps->ctb_size - 1)
+ last_ctb_decoded = 1;
+
+ /* End of substream: last CTB of the tile, or damaged input */
+ if (!ret)
+ t->ended = 1;
+ }
+
+ /* lc still carries the slice of the last column */
+ if (last_ctb_decoded)
+ finish_last_ctb_filter(s, l, lc,
+ (sps->ctb_width - 1) << sps->log2_ctb_size,
+ (sps->ctb_height - 1) << sps->log2_ctb_size);
+
+ return 0;
+}
+
+/* Tiles bound together in one slice by entry points. */
+static int hls_decode_entry_tiles(HEVCContext *s, const H2645NAL *nal)
+{
+ const HEVCLayerContext *const l = &s->layers[s->cur_layer];
+ const HEVCPPS *const pps = s->pps;
+ const HEVCSPS *const sps = pps->sps;
+ const int nb_tiles = pps->num_tile_columns * pps->num_tile_rows;
+ int ret;
+
+ ret = slice_substreams_init(s, nal);
+ if (ret < 0)
+ return ret;
+
+ ret = alloc_substreams(s, pps->num_tile_columns);
+ if (ret < 0)
+ return ret;
+
+ ret = alloc_tile_streams(s, nb_tiles);
+ if (ret < 0)
+ return ret;
+
+ for (int i = 0; i < nb_tiles; i++) {
+ HEVCTileStream *const t = &s->tile_streams[i];
+ const int start_ts = pps->ctb_addr_rs_to_ts[pps->tile_pos_rs[i]];
+
+ if (s->sh.size[i] <= 0 || pps->tile_id[start_ts] != i)
+ return AVERROR_INVALIDDATA;
+
+ t->sh = &s->sh;
+ t->rpl = s->cur_frame->refPicList;
+ t->collocated_ref = s->collocated_ref;
+ t->data = s->data + s->sh.offset[i];
+ t->size = s->sh.size[i];
+ t->start_ts = start_ts;
+ /* The QP seed is the value decode_slice_data() left in lc[0]; with
+ * cu_qp_delta enabled it must not be overwritten with slice_qp */
+ t->qp_y_seed = s->local_ctx[0].sub.qp_y;
+ }
+
+ ret = decode_tiles_row_major(s, l, s->tile_streams);
+ if (ret < 0)
+ return ret;
+
+ return sps->ctb_size;
+}
+
static int hls_slice_data_wpp(HEVCContext *s, const H2645NAL *nal)
{
const HEVCPPS *const pps = s->pps;
@@ -3279,6 +3477,13 @@ static int decode_slice_data(HEVCContext *s, const HEVCLayerContext *l,
s->sh.num_entry_point_offsets + 1 == pps->num_tile_rows * pps->num_tile_columns)
return hls_slice_data_tiles(s, nal);
+ if (tile_interleave_wanted(s, pps) &&
+ s->sh.first_slice_in_pic_flag &&
+ !s->sh.dependent_slice_segment_flag &&
+ s->sh.num_entry_point_offsets ==
+ pps->num_tile_columns * pps->num_tile_rows - 1)
+ return hls_decode_entry_tiles(s, nal);
+
return hls_decode_entry(s, gb);
}
@@ -4259,6 +4464,8 @@ static av_cold int hevc_decode_free(AVCodecContext *avctx)
av_freep(&s->sh.size);
av_freep(&s->local_ctx);
+ av_freep(&s->tile_streams);
+ av_freep(&s->substreams);
ff_h2645_packet_uninit(&s->pkt);
@@ -4508,6 +4715,9 @@ static av_cold void hevc_decode_flush(AVCodecContext *avctx)
static const AVOption options[] = {
{ "apply_defdispwin", "Apply default display window from VUI", OFFSET(apply_defdispwin),
AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, PAR },
+ { "tile_interleave", "Decode tiled pictures in CTB row order to keep frame threading overlapped "
+ "(-1: auto, 0: off, 1: force)", OFFSET(tile_interleave),
+ AV_OPT_TYPE_INT, {.i64 = -1}, -1, 1, PAR },
{ "strict-displaywin", "strictly apply default display window size", OFFSET(apply_defdispwin),
AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, PAR },
{ "view_ids", "Array of view IDs that should be decoded and output; a single -1 to decode all views",
diff --git a/libavcodec/hevc/hevcdec.h b/libavcodec/hevc/hevcdec.h
index 27efa4d012..8c4d65f1a0 100644
--- a/libavcodec/hevc/hevcdec.h
+++ b/libavcodec/hevc/hevcdec.h
@@ -502,6 +502,19 @@ typedef struct HEVCLayerContext {
struct AVRefStructPool *rpl_tab_pool;
} HEVCLayerContext;
+/* Start and state of one tile, for tiles bound by entry points and for one
+ * slice per tile alike. */
+typedef struct HEVCTileStream {
+ const SliceHeader *sh; ///< header of the slice this tile belongs to
+ const RefPicList *rpl; ///< reference lists of that slice, NULL for I slices
+ const HEVCFrame *collocated_ref; ///< temporal MVP target of that slice
+ const uint8_t *data; ///< start of the CABAC substream
+ size_t size;
+ int qp_y_seed;
+ int start_ts; ///< first CTB of the tile, in tile scan order
+ int ended; ///< the substream signalled its end (damaged input)
+} HEVCTileStream;
+
typedef struct HEVCContext {
const AVClass *c; // needed by private avoptions
AVCodecContext *avctx;
@@ -600,6 +613,16 @@ typedef struct HEVCContext {
AVBufferRef *rpu_buf; ///< 0 or 1 Dolby Vision RPUs.
DOVIContext dovi_ctx; ///< Dolby Vision decoding context
+
+ /* Per-tile substream table of the picture, one entry per tile */
+ HEVCTileStream *tile_streams;
+ unsigned tile_streams_alloc;
+
+ /* Substream state per tile column, only ever grows */
+ HEVCSubstream *substreams;
+ unsigned nb_substreams;
+
+ int tile_interleave;
} HEVCContext;
/**
diff --git a/libavcodec/version.h b/libavcodec/version.h
index 26ee41eb1f..e99bd317e3 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -30,7 +30,7 @@
#include "version_major.h"
#define LIBAVCODEC_VERSION_MINOR 26
-#define LIBAVCODEC_VERSION_MICRO 100
+#define LIBAVCODEC_VERSION_MICRO 101
#define LIBAVCODEC_VERSION_INT AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
LIBAVCODEC_VERSION_MINOR, \