Commit f29e8dd79e for ffmpeg

commit f29e8dd79eed86a73f6fce67e1d9776633c1db07
Author: zhang xingxing <2388969553@qq.com>
Date:   Tue Oct 13 06:39:23 2026 +0200

    avformat/rtmpdh: fail bn_hex2bn() when bn_new() fails with GMP and mbedTLS

    This is not a security issue.

    Fixes: d9RoImPBG3x8
    Fixes: swarm-NULL-q1-passed-to-bn_modexp-when-GMP-MbedTLS-alloc-fails-c4db13
    Regression since: 78efc69e7c (GMP), 62f5c9d68b (mbedTLS)
    NULL bignum passed to bn_modexp() Replicated through Modified FFmpeg with ASAN (GMP and mbedTLS builds, the q1 allocation in ff_dh_generate_public_key() made to fail)
    Found-by: zhang xingxing <2388969553@qq.com>
    Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

diff --git a/libavformat/rtmpdh.c b/libavformat/rtmpdh.c
index 6a6c2ccd87..fb7aaf07c0 100644
--- a/libavformat/rtmpdh.c
+++ b/libavformat/rtmpdh.c
@@ -95,7 +95,7 @@
         if (bn)                                     \
             ret = (mpz_set_str(bn, buf, 16) == 0);  \
         else                                        \
-            ret = 1;                                \
+            ret = 0;                                \
     } while (0)
 #define bn_random(bn, num_bits)                       \
     do {                                              \
@@ -198,7 +198,7 @@ static int bn_modexp(FFBigNum bn, FFBigNum y, FFBigNum q, FFBigNum p)
         if (bn)                                     \
             ret = (mbedtls_mpi_read_string(bn, 16, buf) == 0);  \
         else                                        \
-            ret = 1;                                \
+            ret = 0;                                \
     } while (0)
 #define bn_random(bn, num_bits)                     \
     do {                                            \