Commit 439b452f9f for ffmpeg

commit 439b452f9fe916a9f7d8788ab6afee8fac846b0b
Author: Michael Niedermayer <michael@niedermayer.cc>
Date:   Sat Oct 24 06:38:34 2026 +0200

    avcodec/proresenc_kostya: compute the frame size upper bound in 64 bits

    Signed integer overflow Replicated through UnModified FFmpeg with UBSAN
    Out of array write Replicated through UnModified FFmpeg with ASAN

    Fixes: out of array write
    Fixes: EMbkV19WJjmA
    Fixes: AISLE-2026-0150-00821

    Found-by: Joshua Rogers (AISLE)
    Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavcodec/proresenc_kostya.c b/libavcodec/proresenc_kostya.c
index 65346ca170..bf7c4bda43 100644
--- a/libavcodec/proresenc_kostya.c
+++ b/libavcodec/proresenc_kostya.c
@@ -762,12 +762,9 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     int frame_size, picture_size, slice_size;
     int pkt_size, ret;
     int max_slice_size = (ctx->frame_size_upper_bound - 200) / (ctx->pictures_per_frame * ctx->slices_per_picture + 1);
-    int slice_size_upper_bound = ctx->mbs_per_slice * (ctx->chroma_factor == CFACTOR_Y444 ? 768 : 512) * 8;
+    int slice_size_upper_bound = ctx->slice_size_upper_bound;
     uint8_t frame_flags;

-    if (ctx->alpha_bits)
-        slice_size_upper_bound += ctx->mbs_per_slice * 256 * (1 + ctx->alpha_bits + 1) + 7 >> 3;
-
     ctx->pic = pic;
     pkt_size = ctx->frame_size_upper_bound + slice_size_upper_bound;

diff --git a/libavcodec/proresenc_kostya_common.c b/libavcodec/proresenc_kostya_common.c
index 558637d144..173268fff8 100644
--- a/libavcodec/proresenc_kostya_common.c
+++ b/libavcodec/proresenc_kostya_common.c
@@ -24,6 +24,7 @@
 #include "libavutil/pixdesc.h"
 #include "avcodec.h"
 #include "bytestream.h"
+#include "encode.h"
 #include "proresdata.h"
 #include <sys/types.h>
 #include "proresenc_kostya_common.h"
@@ -283,20 +284,28 @@ av_cold int ff_prores_kostya_encode_init(AVCodecContext *avctx, ProresContext *c
             ctx->bits_per_mb += ls_chroma * 4;
     }

-    ctx->frame_size_upper_bound = (ctx->pictures_per_frame *
+    int64_t frame_size_upper_bound = (int64_t)(ctx->pictures_per_frame *
                                    ctx->slices_per_picture + 1) *
                                   (2 + 2 * ctx->num_planes +
                                    (mps * ctx->bits_per_mb) / 8)
                                   + 200;
+    ctx->slice_size_upper_bound = mps * (ctx->chroma_factor == CFACTOR_Y444 ? 768 : 512) * 8;

     if (ctx->alpha_bits) {
          // The alpha plane is run-coded and might exceed the bit budget.
-         ctx->frame_size_upper_bound += (ctx->pictures_per_frame *
-                                         ctx->slices_per_picture + 1) *
-         /* num pixels per slice */     (ctx->mbs_per_slice * 256 *
-         /* bits per pixel */            (1 + ctx->alpha_bits + 1) + 7 >> 3);
+         int alpha_slice_size = /* num pixels per slice */ mps * 256 *
+                                /* bits per pixel */       (1 + ctx->alpha_bits + 1) + 7 >> 3;
+         frame_size_upper_bound += (int64_t)(ctx->pictures_per_frame *
+                                             ctx->slices_per_picture + 1) * alpha_slice_size;
+         ctx->slice_size_upper_bound += alpha_slice_size;
     }

+    if (frame_size_upper_bound + ctx->slice_size_upper_bound + FF_INPUT_BUFFER_MIN_SIZE > INT_MAX) {
+        av_log(avctx, AV_LOG_ERROR, "Frame too large to encode\n");
+        return AVERROR(EINVAL);
+    }
+    ctx->frame_size_upper_bound = frame_size_upper_bound;
+
     avctx->codec_tag   = ctx->profile_info->tag;
     avctx->profile = ctx->profile;

diff --git a/libavcodec/proresenc_kostya_common.h b/libavcodec/proresenc_kostya_common.h
index ed5355db0f..8055394a72 100644
--- a/libavcodec/proresenc_kostya_common.h
+++ b/libavcodec/proresenc_kostya_common.h
@@ -112,6 +112,7 @@ typedef struct ProresContext {
     int quant_sel;

     int frame_size_upper_bound;
+    int slice_size_upper_bound;

     int profile;
     const struct prores_profile *profile_info;