Commit 738961c650 for ffmpeg
commit 738961c650a8724e826ce0232715d2d5fc5b703f
Author: Piotr Niełacny <piotr.nielacny@gmail.com>
Date: Mon Apr 13 18:50:08 2026 +0200
fftools/ffmpeg_sched: track upstream timestamp offsets for pacing
With output-side -ss, audio/video filtering and subtitle encoding
subtract the output start time before packets reach the scheduler.
Streamcopy performs its subtraction later, in the muxer. Comparing
these different timelines can starve copied audio long enough for early
HLS segments to be written without audio packets.
Register the upstream timestamp offset for each encoded mux stream and
add it back in a separate last_dts_paced field. Use that field for both
the trailing timestamp and choke comparisons, while retaining the raw
last_dts for progress reporting. Register subtitle offsets as well,
matching the packet timestamp rebasing introduced by 7f2b4adeac954.
Add a generated-input HLS FATE test using native encoders. Declare the
source, transcode and probe dependencies and register the test with
FATE_FFMPEG_FFPROBE. Use a single stream listing in the reference, as
required after 88f7fd9ec525 made program-stream output explicit.
The scheduler regression was introduced by d119ae2fd82a.
Fixes #22765.
Assisted-by: Fairy
Signed-off-by: Piotr Niełacny <piotr.nielacny@gmail.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
diff --git a/fftools/ffmpeg_mux_init.c b/fftools/ffmpeg_mux_init.c
index 15b436cc56..00d33bb03b 100644
--- a/fftools/ffmpeg_mux_init.c
+++ b/fftools/ffmpeg_mux_init.c
@@ -1635,6 +1635,12 @@ static int ost_add(Muxer *mux, const OptionsContext *o, enum AVMediaType type,
}
}
+ // Audio/video filters and subtitle encoding subtract the output start
+ // time before scheduling; streamcopy does so later in the muxer.
+ if (ost->enc && mux->of.start_time != AV_NOPTS_VALUE && ms->sch_idx >= 0)
+ sch_mux_stream_set_dts_pacing_offset(mux->sch, mux->sch_idx,
+ ms->sch_idx, mux->of.start_time);
+
if (src.type != SCH_NODE_TYPE_NONE) {
ret = sch_connect(mux->sch,
src, SCH_MSTREAM(mux->sch_idx, ms->sch_idx));
diff --git a/fftools/ffmpeg_sched.c b/fftools/ffmpeg_sched.c
index e9b57ed62b..fe64eb6557 100644
--- a/fftools/ffmpeg_sched.c
+++ b/fftools/ffmpeg_sched.c
@@ -198,12 +198,22 @@ typedef struct SchMuxStream {
// an EOF was generated while flushing the pre-mux queue
int init_eof;
+ // offset (in AV_TIME_BASE_Q) subtracted by the upstream pipeline
+ // (e.g. filter ts_offset), added back when computing last_dts_paced
+ // so that cross-stream pacing compares on a common source timeline
+ int64_t dts_pacing_offset;
+
////////////////////////////////////////////////////////////
// The following are protected by Scheduler.schedule_lock //
/* dts+duration of the last packet sent to this stream
- in AV_TIME_BASE_Q */
+ in AV_TIME_BASE_Q, as observed by the muxer (no offset applied);
+ feeds progressing_dts() -> sch->last_dts -> CLI progress display */
int64_t last_dts;
+ /* last_dts + dts_pacing_offset, kept on a common source timeline
+ across streams; consumed by trailing_dts() and the choke check
+ in schedule_update_locked() */
+ int64_t last_dts_paced;
// this stream no longer accepts input
int source_finished;
////////////////////////////////////////////////////////////
@@ -447,10 +457,10 @@ static int64_t trailing_dts(const Scheduler *sch)
if (ms->source_finished)
continue;
- if (ms->last_dts == AV_NOPTS_VALUE)
+ if (ms->last_dts_paced == AV_NOPTS_VALUE)
return AV_NOPTS_VALUE;
- min_dts = FFMIN(min_dts, ms->last_dts);
+ min_dts = FFMIN(min_dts, ms->last_dts_paced);
}
}
@@ -704,7 +714,8 @@ int sch_add_mux_stream(Scheduler *sch, unsigned mux_idx)
if (!ms->pre_mux_queue.fifo)
return AVERROR(ENOMEM);
- ms->last_dts = AV_NOPTS_VALUE;
+ ms->last_dts = AV_NOPTS_VALUE;
+ ms->last_dts_paced = AV_NOPTS_VALUE;
return stream_idx;
}
@@ -1248,6 +1259,21 @@ void sch_mux_stream_buffering(Scheduler *sch, unsigned mux_idx, unsigned stream_
ms->pre_mux_queue.data_threshold = data_threshold;
}
+void sch_mux_stream_set_dts_pacing_offset(Scheduler *sch, unsigned mux_idx,
+ unsigned stream_idx, int64_t offset)
+{
+ SchMux *mux;
+ SchMuxStream *ms;
+
+ av_assert0(mux_idx < sch->nb_mux);
+ mux = &sch->mux[mux_idx];
+
+ av_assert0(stream_idx < mux->nb_streams);
+ ms = &mux->streams[stream_idx];
+
+ ms->dts_pacing_offset = offset;
+}
+
int sch_mux_stream_ready(Scheduler *sch, unsigned mux_idx, unsigned stream_idx)
{
SchMux *mux;
@@ -1457,9 +1483,9 @@ static void schedule_update_locked(Scheduler *sch)
// and not too far ahead of the trailing stream
if (ms->source_finished)
continue;
- if (dts == AV_NOPTS_VALUE && ms->last_dts != AV_NOPTS_VALUE)
+ if (dts == AV_NOPTS_VALUE && ms->last_dts_paced != AV_NOPTS_VALUE)
continue;
- if (dts != AV_NOPTS_VALUE && ms->last_dts - dts >= SCHEDULE_TOLERANCE)
+ if (dts != AV_NOPTS_VALUE && ms->last_dts_paced - dts >= SCHEDULE_TOLERANCE)
continue;
// resolve the source to unchoke
@@ -2053,8 +2079,12 @@ update_schedule:
if (dts != AV_NOPTS_VALUE || !pkt) {
pthread_mutex_lock(&sch->schedule_lock);
- if (pkt) ms->last_dts = dts;
- else ms->source_finished = 1;
+ if (pkt) {
+ ms->last_dts = dts;
+ ms->last_dts_paced = dts + ms->dts_pacing_offset;
+ } else {
+ ms->source_finished = 1;
+ }
schedule_update_locked(sch);
diff --git a/fftools/ffmpeg_sched.h b/fftools/ffmpeg_sched.h
index 952e56e7b5..26f8fb6d94 100644
--- a/fftools/ffmpeg_sched.h
+++ b/fftools/ffmpeg_sched.h
@@ -285,6 +285,14 @@ int sch_add_mux_stream(Scheduler *sch, unsigned mux_idx);
void sch_mux_stream_buffering(Scheduler *sch, unsigned mux_idx, unsigned stream_idx,
size_t data_threshold, int max_packets);
+/**
+ * Set the timestamp offset (in AV_TIME_BASE_Q) that the upstream pipeline
+ * subtracted from this stream's packets. The scheduler adds it back when
+ * comparing DTS across streams for pacing. Defaults to 0 (streamcopy).
+ */
+void sch_mux_stream_set_dts_pacing_offset(Scheduler *sch, unsigned mux_idx,
+ unsigned stream_idx, int64_t offset);
+
/**
* Signal to the scheduler that the specified muxed stream is initialized and
* ready. Muxing is started once all the streams are ready.
diff --git a/tests/fate/hlsenc.mak b/tests/fate/hlsenc.mak
index 26b029caf0..b52d306726 100644
--- a/tests/fate/hlsenc.mak
+++ b/tests/fate/hlsenc.mak
@@ -265,7 +265,58 @@ fate-hls-fmp4-big-init: tests/data/hls_fmp4_big_init.m3u8
fate-hls-fmp4-big-init: CLEANFILES = tests/data/hls_fmp4_big_init.m3u8 tests/data/hls_fmp4_big_init.mp4 tests/data/hls_fmp4_big_init_*.m4s
fate-hls-fmp4-big-init: CMD = framecrc -i $(TARGET_PATH)/tests/data/hls_fmp4_big_init.m3u8 -map 0:a:99 -c copy
+# Regression test for cross-stream pacing with output -ss. The transcoded
+# video's timestamps are rebased before the scheduler observes them, whereas
+# copied audio is rebased by of_streamcopy() after the scheduler observes it.
+# Comparing these different timelines can starve audio in early HLS segments.
+tests/data/hls_sched_v.mkv: TAG = GEN
+tests/data/hls_sched_v.mkv: ffmpeg$(PROGSSUF)$(EXESUF) | tests/data
+ $(M)$(TARGET_EXEC) $(TARGET_PATH)/$< -nostdin \
+ -f lavfi -i "testsrc2=size=128x72:rate=24:d=180" \
+ -c:v mpeg4 -g 500 -flags +bitexact \
+ -an -y $(TARGET_PATH)/tests/data/hls_sched_v.mkv 2>/dev/null
+
+tests/data/hls_sched_a.mka: TAG = GEN
+tests/data/hls_sched_a.mka: ffmpeg$(PROGSSUF)$(EXESUF) | tests/data
+ $(M)$(TARGET_EXEC) $(TARGET_PATH)/$< -nostdin \
+ -f lavfi -i "aevalsrc=sin(2*PI*440*t):d=180:sample_rate=44100" \
+ -c:a mp2fixed -b:a 128k \
+ -vn -y $(TARGET_PATH)/tests/data/hls_sched_a.mka 2>/dev/null
+
+tests/data/hls_sched_extaudio.m3u8: TAG = GEN
+tests/data/hls_sched_extaudio.m3u8: ffmpeg$(PROGSSUF)$(EXESUF) tests/data/hls_sched_v.mkv tests/data/hls_sched_a.mka | tests/data
+ $(M)$(TARGET_EXEC) $(TARGET_PATH)/$< -nostdin \
+ -ss 120 -noaccurate_seek -i $(TARGET_PATH)/tests/data/hls_sched_v.mkv \
+ -ss 120 -noaccurate_seek -i $(TARGET_PATH)/tests/data/hls_sched_a.mka \
+ -ss 120 -map 0:v -map 1:a -c:v mpeg2video -flags +bitexact \
+ -c:a copy -copyts -avoid_negative_ts disabled \
+ -f hls -hls_time 2 -hls_list_size 0 \
+ -hls_segment_filename $(TARGET_PATH)/tests/data/hls_sched_extaudio_%d.ts \
+ $(TARGET_PATH)/tests/data/hls_sched_extaudio.m3u8 2>/dev/null
+
+FATE_HLSENC_FFMPEG_FFPROBE-$(call ALLYES, TESTSRC2_FILTER AEVALSRC_FILTER ARESAMPLE_FILTER LAVFI_INDEV \
+ MPEG4_ENCODER MPEG2VIDEO_ENCODER MP2FIXED_ENCODER \
+ WRAPPED_AVFRAME_DECODER MPEG4_DECODER MPEG2VIDEO_DECODER MP2_DECODER \
+ $(if $(HAVE_BIGENDIAN),PCM_F64BE_DECODER,PCM_F64LE_DECODER) \
+ MPEG4VIDEO_PARSER MPEGVIDEO_PARSER MPEGAUDIO_PARSER \
+ MATROSKA_MUXER MATROSKA_AUDIO_MUXER MATROSKA_DEMUXER MPEGTS_DEMUXER \
+ HLS_MUXER MPEGTS_MUXER FILE_PROTOCOL) += fate-hls-sched-extaudio
+fate-hls-sched-extaudio: tests/data/hls_sched_extaudio.m3u8
+fate-hls-sched-extaudio: CLEANFILES = tests/data/hls_sched_extaudio.m3u8 tests/data/hls_sched_extaudio_*.ts tests/data/hls_sched_v.mkv tests/data/hls_sched_a.mka
+# Probe the first HLS segment's audio stream parameters. Without the fix,
+# audio is starved during the segment's window and only sample_rate=0,
+# channels=0 are reported (the MPEG-TS PSI declares the codec but no audio
+# packets are present). With the fix, audio reaches the segment in time.
+fate-hls-sched-extaudio: CMD = run ffprobe$(PROGSSUF)$(EXESUF) -bitexact -threads $(THREADS) \
+ -analyzeduration 10000000 -probesize 10000000 \
+ -show_entries stream=codec_type,codec_name,sample_rate,channels \
+ -print_format default=nw=1 \
+ $(TARGET_PATH)/tests/data/hls_sched_extaudio_0.ts
+fate-hls-sched-extaudio: CMP = diff
+
FATE_HLSENC_LAVFI-yes := $(if $(call FRAMECRC), $(FATE_HLSENC_LAVFI-yes))
+FATE_FFMPEG_FFPROBE += $(FATE_HLSENC_FFMPEG_FFPROBE-yes)
+
FATE_FFMPEG += $(FATE_HLSENC_LAVFI-yes)
-fate-hlsenc: $(FATE_HLSENC-yes) $(FATE_HLSENC_PROBE-yes) $(FATE_HLSENC_LAVFI-yes)
+fate-hlsenc: $(FATE_HLSENC-yes) $(FATE_HLSENC_PROBE-yes) $(FATE_HLSENC_LAVFI-yes) $(FATE_HLSENC_FFMPEG_FFPROBE-yes)
diff --git a/tests/ref/fate/hls-sched-extaudio b/tests/ref/fate/hls-sched-extaudio
new file mode 100644
index 0000000000..d13d84963c
--- /dev/null
+++ b/tests/ref/fate/hls-sched-extaudio
@@ -0,0 +1,6 @@
+codec_name=mpeg2video
+codec_type=video
+codec_name=mp2
+codec_type=audio
+sample_rate=44100
+channels=1