Commit 2b99c068db for ffmpeg
commit 2b99c068db09fb149330421d88999486fce83ed8
Author: Michael Niedermayer <michael@niedermayer.cc>
Date: Wed Oct 7 00:16:06 2026 +0200
avfilter/yadif: propagate errorsl
Not a security issue. Not tested on VideoToolbox.
Unprocessed output frames Replicated through UnModified FFmpeg with CUDA and Vulkan call failures injected by gdb
Found-by: Joshua Rogers <joshua.rogers@aisle.com>
Suggested-by: Joshua Rogers <joshua.rogers@aisle.com>
diff --git a/libavfilter/vf_bwdif.c b/libavfilter/vf_bwdif.c
index 5fba0e7a14..f7f715f3cb 100644
--- a/libavfilter/vf_bwdif.c
+++ b/libavfilter/vf_bwdif.c
@@ -108,7 +108,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
return 0;
}
-static void filter(AVFilterContext *ctx, AVFrame *dstpic,
+static int filter(AVFilterContext *ctx, AVFrame *dstpic,
int parity, int tff)
{
BWDIFContext *bwdif = ctx->priv;
@@ -135,6 +135,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dstpic,
if (yadif->current_field == YADIF_FIELD_END) {
yadif->current_field = YADIF_FIELD_NORMAL;
}
+
+ return 0;
}
static const enum AVPixelFormat pix_fmts[] = {
diff --git a/libavfilter/vf_bwdif_cuda.c b/libavfilter/vf_bwdif_cuda.c
index cc954ab1d3..9879d12ab4 100644
--- a/libavfilter/vf_bwdif_cuda.c
+++ b/libavfilter/vf_bwdif_cuda.c
@@ -53,7 +53,7 @@ typedef struct DeintCUDAContext {
#define CHECK_CU(x) FF_CUDA_CHECK_DL(ctx, s->hwctx->internal->cuda_dl, x)
-static CUresult call_kernel(AVFilterContext *ctx, CUfunction func,
+static int call_kernel(AVFilterContext *ctx, CUfunction func,
CUdeviceptr prev, CUdeviceptr cur, CUdeviceptr next,
CUarray_format format, int channels,
int src_width, // Width is pixels per channel
@@ -122,7 +122,7 @@ exit:
return ret;
}
-static void filter(AVFilterContext *ctx, AVFrame *dst,
+static int filter(AVFilterContext *ctx, AVFrame *dst,
int parity, int tff)
{
DeintCUDAContext *s = ctx->priv;
@@ -133,7 +133,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
ret = CHECK_CU(cu->cuCtxPushCurrent(s->hwctx->cuda_ctx));
if (ret < 0)
- return;
+ return ret;
for (i = 0; i < y->csp->nb_components; i++) {
CUfunction func;
@@ -151,6 +151,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
channels = comp->step / pixel_size;
if (pixel_size > 2 || channels > 2) {
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
+ ret = AVERROR(EINVAL);
goto exit;
}
switch (pixel_size) {
@@ -164,6 +165,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
break;
default:
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
+ ret = AVERROR(EINVAL);
goto exit;
}
@@ -172,7 +174,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
av_log(ctx, AV_LOG_TRACE,
"Deinterlacing plane %d: pixel_size: %d channels: %d\n",
comp->plane, pixel_size, channels);
- call_kernel(ctx, func,
+ ret = call_kernel(ctx, func,
(CUdeviceptr)y->prev->data[i],
(CUdeviceptr)y->cur->data[i],
(CUdeviceptr)y->next->data[i],
@@ -185,6 +187,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
AV_CEIL_RSHIFT(dst->height, i ? y->csp->log2_chroma_h : 0),
dst->linesize[i] / comp->step,
parity, tff, clip_max);
+ if (ret < 0)
+ goto exit;
}
if (y->current_field == YADIF_FIELD_END) {
@@ -193,7 +197,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
exit:
CHECK_CU(cu->cuCtxPopCurrent(&dummy));
- return;
+ return ret;
}
static av_cold void deint_cuda_uninit(AVFilterContext *ctx)
diff --git a/libavfilter/vf_bwdif_vulkan.c b/libavfilter/vf_bwdif_vulkan.c
index 2c1e4908ef..5f96d2f7a3 100644
--- a/libavfilter/vf_bwdif_vulkan.c
+++ b/libavfilter/vf_bwdif_vulkan.c
@@ -102,7 +102,7 @@ fail:
return err;
}
-static void bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst,
+static int bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst,
int parity, int tff)
{
BWDIFVulkanContext *s = ctx->priv;
@@ -112,13 +112,16 @@ static void bwdif_vulkan_filter_frame(AVFilterContext *ctx, AVFrame *dst,
.tff = tff,
.current_field = y->current_field,
};
+ int ret;
- ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->shd, dst,
+ ret = ff_vk_filter_process_Nin(&s->vkctx, &s->e, &s->shd, dst,
(AVFrame *[]){ y->prev, y->cur, y->next }, 3,
VK_NULL_HANDLE, 1, ¶ms, sizeof(params));
if (y->current_field == YADIF_FIELD_END)
y->current_field = YADIF_FIELD_NORMAL;
+
+ return ret;
}
static void bwdif_vulkan_uninit(AVFilterContext *avctx)
diff --git a/libavfilter/vf_yadif.c b/libavfilter/vf_yadif.c
index 41ba056ad4..6fe9a97a13 100644
--- a/libavfilter/vf_yadif.c
+++ b/libavfilter/vf_yadif.c
@@ -225,7 +225,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
return 0;
}
-static void filter(AVFilterContext *ctx, AVFrame *dstpic,
+static int filter(AVFilterContext *ctx, AVFrame *dstpic,
int parity, int tff)
{
YADIFContext *yadif = ctx->priv;
@@ -249,6 +249,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dstpic,
ff_filter_execute(ctx, filter_slice, &td, NULL,
FFMIN(h, ff_filter_get_nb_threads(ctx)));
}
+
+ return 0;
}
static const enum AVPixelFormat pix_fmts[] = {
diff --git a/libavfilter/vf_yadif_cuda.c b/libavfilter/vf_yadif_cuda.c
index 50ac61ad8a..435657de5d 100644
--- a/libavfilter/vf_yadif_cuda.c
+++ b/libavfilter/vf_yadif_cuda.c
@@ -53,7 +53,7 @@ typedef struct DeintCUDAContext {
#define CHECK_CU(x) FF_CUDA_CHECK_DL(ctx, s->hwctx->internal->cuda_dl, x)
-static CUresult call_kernel(AVFilterContext *ctx, CUfunction func,
+static int call_kernel(AVFilterContext *ctx, CUfunction func,
CUdeviceptr prev, CUdeviceptr cur, CUdeviceptr next,
CUarray_format format, int channels,
int src_width, // Width is pixels per channel
@@ -121,7 +121,7 @@ exit:
return ret;
}
-static void filter(AVFilterContext *ctx, AVFrame *dst,
+static int filter(AVFilterContext *ctx, AVFrame *dst,
int parity, int tff)
{
DeintCUDAContext *s = ctx->priv;
@@ -132,7 +132,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
ret = CHECK_CU(cu->cuCtxPushCurrent(s->hwctx->cuda_ctx));
if (ret < 0)
- return;
+ return ret;
for (i = 0; i < y->csp->nb_components; i++) {
CUfunction func;
@@ -150,6 +150,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
channels = comp->step / pixel_size;
if (pixel_size > 2 || channels > 2) {
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
+ ret = AVERROR(EINVAL);
goto exit;
}
switch (pixel_size) {
@@ -163,12 +164,13 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
break;
default:
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
+ ret = AVERROR(EINVAL);
goto exit;
}
av_log(ctx, AV_LOG_TRACE,
"Deinterlacing plane %d: pixel_size: %d channels: %d\n",
comp->plane, pixel_size, channels);
- call_kernel(ctx, func,
+ ret = call_kernel(ctx, func,
(CUdeviceptr)y->prev->data[i],
(CUdeviceptr)y->cur->data[i],
(CUdeviceptr)y->next->data[i],
@@ -181,11 +183,13 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
AV_CEIL_RSHIFT(dst->height, i ? y->csp->log2_chroma_h : 0),
dst->linesize[i] / comp->step,
parity, tff);
+ if (ret < 0)
+ goto exit;
}
exit:
CHECK_CU(cu->cuCtxPopCurrent(&dummy));
- return;
+ return ret;
}
static av_cold void deint_cuda_uninit(AVFilterContext *ctx)
diff --git a/libavfilter/vf_yadif_videotoolbox.m b/libavfilter/vf_yadif_videotoolbox.m
index 9f09947cc2..b1152987cd 100644
--- a/libavfilter/vf_yadif_videotoolbox.m
+++ b/libavfilter/vf_yadif_videotoolbox.m
@@ -100,7 +100,7 @@ static void call_kernel(AVFilterContext *ctx,
ff_objc_release(&buffer);
}
-static void filter(AVFilterContext *ctx, AVFrame *dst,
+static int filter(AVFilterContext *ctx, AVFrame *dst,
int parity, int tff) API_AVAILABLE(macos(10.11), ios(8.0))
{
YADIFVTContext *s = ctx->priv;
@@ -123,7 +123,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
channels = comp->step / pixel_size;
if (pixel_size > 2 || channels > 2) {
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
- return;
+ return AVERROR(EINVAL);
}
switch (pixel_size) {
case 1:
@@ -134,7 +134,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
break;
default:
av_log(ctx, AV_LOG_ERROR, "Unsupported pixel format: %s\n", y->csp->name);
- return;
+ return AVERROR(EINVAL);
}
av_log(ctx, AV_LOG_TRACE,
"Deinterlacing plane %d: pixel_size: %d channels: %d\n",
@@ -151,7 +151,7 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
if (cur) CFRelease(cur);
if (next) CFRelease(next);
if (dest) CFRelease(dest);
- return;
+ return AVERROR_EXTERNAL;
}
tex_prev = CVMetalTextureGetTexture(prev);
@@ -173,6 +173,8 @@ static void filter(AVFilterContext *ctx, AVFrame *dst,
if (y->current_field == YADIF_FIELD_END) {
y->current_field = YADIF_FIELD_NORMAL;
}
+
+ return 0;
}
static av_cold void do_uninit(AVFilterContext *ctx) API_AVAILABLE(macos(10.11), ios(8.0))
diff --git a/libavfilter/yadif.h b/libavfilter/yadif.h
index 888ba12365..21c6df29c2 100644
--- a/libavfilter/yadif.h
+++ b/libavfilter/yadif.h
@@ -62,7 +62,7 @@ typedef struct YADIFContext {
AVFrame *prev;
AVFrame *out;
- void (*filter)(AVFilterContext *ctx, AVFrame *dstpic, int parity, int tff);
+ int (*filter)(AVFilterContext *ctx, AVFrame *dstpic, int parity, int tff);
/**
* Required alignment for filter_line
diff --git a/libavfilter/yadif_common.c b/libavfilter/yadif_common.c
index a6cd981684..2b1c9bc1cd 100644
--- a/libavfilter/yadif_common.c
+++ b/libavfilter/yadif_common.c
@@ -49,7 +49,11 @@ static int return_frame(AVFilterContext *ctx, int is_second)
yadif->current_field = YADIF_FIELD_END;
}
- yadif->filter(ctx, yadif->out, tff ^ !is_second, tff);
+ ret = yadif->filter(ctx, yadif->out, tff ^ !is_second, tff);
+ if (ret < 0) {
+ av_frame_free(&yadif->out);
+ return ret;
+ }
if (is_second) {
int64_t cur_pts = yadif->cur->pts;
@@ -100,13 +104,19 @@ int ff_yadif_filter_frame(AVFilterLink *link, AVFrame *frame)
{
AVFilterContext *ctx = link->dst;
YADIFContext *yadif = ctx->priv;
+ int ret;
av_assert0(frame);
ff_ccfifo_extract(&yadif->cc_fifo, frame);
- if (yadif->frame_pending)
- return_frame(ctx, 1);
+ if (yadif->frame_pending) {
+ ret = return_frame(ctx, 1);
+ if (ret < 0) {
+ av_frame_free(&frame);
+ return ret;
+ }
+ }
if (yadif->prev)
av_frame_free(&yadif->prev);
@@ -177,10 +187,8 @@ int ff_yadif_request_frame(AVFilterLink *link)
YADIFContext *yadif = ctx->priv;
int ret;
- if (yadif->frame_pending) {
- return_frame(ctx, 1);
- return 0;
- }
+ if (yadif->frame_pending)
+ return return_frame(ctx, 1);
if (yadif->eof)
return AVERROR_EOF;
@@ -196,13 +204,11 @@ int ff_yadif_request_frame(AVFilterLink *link)
yadif->current_field = YADIF_FIELD_BACK_END;
next->pts = yadif->next->pts * 2 - yadif->cur->pts;
- ff_yadif_filter_frame(ctx->inputs[0], next);
+ ret = ff_yadif_filter_frame(ctx->inputs[0], next);
yadif->eof = 1;
- } else if (ret < 0) {
- return ret;
}
- return 0;
+ return ret;
}
int ff_yadif_config_output_common(AVFilterLink *outlink)