Commit 4353f63c7c for ffmpeg

commit 4353f63c7c94188d02800ff1b13e27297e4888a5
Author: Michael Niedermayer <michael@niedermayer.cc>
Date:   Mon Oct 19 02:13:33 2026 +0200

    avformat/rtpenc: Reject packet sizes too small for the payload header

    Fixes: negative-size-param
    Fixes: infinite loop
    Fixes: TwYfznAFyatl
    Fixes: #22717
    Fixes: ffmpeg -f lavfi -i testsrc=duration=0.1:size=16x16:rate=1 -c:v libvpx -f rtp -packetsize 13 rtp://127.0.0.1:9999

    Negative-size memcpy and endless loop Replicated through UnModified FFmpeg with ASAN

    Found-by: bird <6666242+bird@users.noreply.github.com>

diff --git a/libavformat/rtpenc.c b/libavformat/rtpenc.c
index 7f09329b10..24bf6b93f8 100644
--- a/libavformat/rtpenc.c
+++ b/libavformat/rtpenc.c
@@ -101,6 +101,7 @@ static int rtp_write_header(AVFormatContext *s1)
 {
     RTPMuxContext *s = s1->priv_data;
     int n, ret = AVERROR(EINVAL);
+    int payload_header_size = 0;
     AVStream *st;

     if (s1->nb_streams != 1) {
@@ -212,12 +213,14 @@ static int rtp_write_header(AVFormatContext *s1)
             ret = AVERROR_EXPERIMENTAL;
             goto fail;
         }
+        payload_header_size = 4;
         break;
     case AV_CODEC_ID_H264:
         /* check for H.264 MP4 syntax */
         if (st->codecpar->extradata_size > 4 && st->codecpar->extradata[0] == 1) {
             s->nal_length_size = (st->codecpar->extradata[4] & 0x03) + 1;
         }
+        payload_header_size = 2;
         break;
     case AV_CODEC_ID_HEVC:
         /* Only check for the standardized hvcC version of extradata, keeping
@@ -227,6 +230,10 @@ static int rtp_write_header(AVFormatContext *s1)
         if (st->codecpar->extradata_size > 21 && st->codecpar->extradata[0] == 1) {
             s->nal_length_size = (st->codecpar->extradata[21] & 0x03) + 1;
         }
+        payload_header_size = 3;
+        break;
+    case AV_CODEC_ID_VP8:
+        payload_header_size = 4;
         break;
     case AV_CODEC_ID_BITPACKED:
     case AV_CODEC_ID_RAWVIDEO:
@@ -307,11 +314,19 @@ static int rtp_write_header(AVFormatContext *s1)
         break;
     case AV_CODEC_ID_AAC:
         s->max_frames_per_packet = 50;
+        if (!(s->flags & FF_RTP_FLAG_MP4A_LATM))
+            payload_header_size = 4;
         break;
     default:
         break;
     }

+    if (s->max_payload_size <= payload_header_size) {
+        av_log(s1, AV_LOG_ERROR, "Max packet size %u too low for a %d byte payload header\n",
+               s1->packet_size, payload_header_size);
+        goto fail;
+    }
+
     return 0;

 fail: