Commit 6a8ea993d1 for ffmpeg
commit 6a8ea993d1f37648e8a88c989102bc35ed696426
Author: Michael Niedermayer <michael@niedermayer.cc>
Date: Sat Oct 10 08:41:49 2026 +0200
libswscale: handle any width in the planar to packed YUV converters
Fixes: out of array read
Fixes: out of array write
Fixes: RTjjU0p47d8Y
Found-by: Ayoub Nabil Boubagrat <237098474+ayoubnabil@users.noreply.github.com>
Out of array read and write Replicated through API with Valgrind and ASAN
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
diff --git a/libswscale/rgb2rgb.h b/libswscale/rgb2rgb.h
index ed6e08a565..6f16f95fa6 100644
--- a/libswscale/rgb2rgb.h
+++ b/libswscale/rgb2rgb.h
@@ -93,32 +93,18 @@ void ff_rgb24toyv12_c(const uint8_t *src, uint8_t *ydst, uint8_t *udst,
uint8_t *vdst, int width, int height, int lumStride,
int chromStride, int srcStride, const int32_t *rgb2yuv);
-/**
- * Height should be a multiple of 2 and width should be a multiple of 16.
- * (If this is a problem for anyone then tell me, and I will fix it.)
- */
extern void (*yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
int width, int height,
int lumStride, int chromStride, int dstStride);
-/**
- * Width should be a multiple of 16.
- */
extern void (*yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
int width, int height,
int lumStride, int chromStride, int dstStride);
-/**
- * Height should be a multiple of 2 and width should be a multiple of 16.
- * (If this is a problem for anyone then tell me, and I will fix it.)
- */
extern void (*yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
int width, int height,
int lumStride, int chromStride, int dstStride);
-/**
- * Width should be a multiple of 16.
- */
extern void (*yuv422ptouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
int width, int height,
int lumStride, int chromStride, int dstStride);
@@ -157,6 +143,22 @@ extern void (*yuyvtoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const u
void ff_sws_rgb2rgb_init(void);
+/**
+ * Packs the chroma positions from..to-1 of one line as YUYV when
+ * luma_first is set and as UYVY otherwise.
+ */
+static inline void ff_pack_yuv422_line(uint8_t *dst, const uint8_t *ysrc,
+ const uint8_t *usrc, const uint8_t *vsrc,
+ int from, int to, int luma_first)
+{
+ for (int i = from; i < to; i++) {
+ dst[4 * i + !luma_first] = ysrc[2 * i];
+ dst[4 * i + 2 + !luma_first] = ysrc[2 * i + 1];
+ dst[4 * i + luma_first] = usrc[i];
+ dst[4 * i + 2 + luma_first] = vsrc[i];
+ }
+}
+
void rgb2rgb_init_aarch64(void);
void rgb2rgb_init_riscv(void);
void rgb2rgb_init_x86(void);
diff --git a/libswscale/rgb2rgb_template.c b/libswscale/rgb2rgb_template.c
index df439783d6..d857b74afb 100644
--- a/libswscale/rgb2rgb_template.c
+++ b/libswscale/rgb2rgb_template.c
@@ -389,7 +389,7 @@ static inline void yuvPlanartoyuy2_c(const uint8_t *ysrc, const uint8_t *usrc,
#if HAVE_FAST_64BIT
uint64_t *ldst = (uint64_t *)dst;
const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
- for (i = 0; i < chromWidth; i += 2) {
+ for (i = 0; i < (chromWidth & ~1); i += 2) {
uint64_t k = yc[0] + (uc[0] << 8) +
(yc[1] << 16) + ((unsigned) vc[0] << 24);
uint64_t l = yc[2] + (uc[1] << 8) +
@@ -399,6 +399,7 @@ static inline void yuvPlanartoyuy2_c(const uint8_t *ysrc, const uint8_t *usrc,
uc += 2;
vc += 2;
}
+ ff_pack_yuv422_line(dst, ysrc, usrc, vsrc, chromWidth & ~1, chromWidth, 1);
#else
int *idst = (int32_t *)dst;
@@ -426,10 +427,6 @@ static inline void yuvPlanartoyuy2_c(const uint8_t *ysrc, const uint8_t *usrc,
}
}
-/**
- * Height should be a multiple of 2 and width should be a multiple of 16.
- * (If this is a problem for anyone then tell me, and I will fix it.)
- */
static inline void yv12toyuy2_c(const uint8_t *ysrc, const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *dst,
int width, int height, int lumStride,
@@ -453,7 +450,7 @@ static inline void yuvPlanartouyvy_c(const uint8_t *ysrc, const uint8_t *usrc,
#if HAVE_FAST_64BIT
uint64_t *ldst = (uint64_t *)dst;
const uint8_t *yc = ysrc, *uc = usrc, *vc = vsrc;
- for (i = 0; i < chromWidth; i += 2) {
+ for (i = 0; i < (chromWidth & ~1); i += 2) {
uint64_t k = uc[0] + (yc[0] << 8) +
(vc[0] << 16) + ((unsigned) yc[1] << 24);
uint64_t l = uc[1] + (yc[2] << 8) +
@@ -463,6 +460,7 @@ static inline void yuvPlanartouyvy_c(const uint8_t *ysrc, const uint8_t *usrc,
uc += 2;
vc += 2;
}
+ ff_pack_yuv422_line(dst, ysrc, usrc, vsrc, chromWidth & ~1, chromWidth, 0);
#else
int *idst = (int32_t *)dst;
@@ -490,10 +488,6 @@ static inline void yuvPlanartouyvy_c(const uint8_t *ysrc, const uint8_t *usrc,
}
}
-/**
- * Height should be a multiple of 2 and width should be a multiple of 16
- * (If this is a problem for anyone then tell me, and I will fix it.)
- */
static inline void yv12touyvy_c(const uint8_t *ysrc, const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *dst,
int width, int height, int lumStride,
@@ -504,9 +498,6 @@ static inline void yv12touyvy_c(const uint8_t *ysrc, const uint8_t *usrc,
chromStride, dstStride, 2);
}
-/**
- * Width should be a multiple of 16.
- */
static inline void yuv422ptouyvy_c(const uint8_t *ysrc, const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *dst,
int width, int height, int lumStride,
@@ -516,9 +507,6 @@ static inline void yuv422ptouyvy_c(const uint8_t *ysrc, const uint8_t *usrc,
chromStride, dstStride, 1);
}
-/**
- * Width should be a multiple of 16.
- */
static inline void yuv422ptoyuy2_c(const uint8_t *ysrc, const uint8_t *usrc,
const uint8_t *vsrc, uint8_t *dst,
int width, int height, int lumStride,
diff --git a/libswscale/x86/rgb2rgb.c b/libswscale/x86/rgb2rgb.c
index 699d341ce9..f22a919575 100644
--- a/libswscale/x86/rgb2rgb.c
+++ b/libswscale/x86/rgb2rgb.c
@@ -1124,46 +1124,100 @@ static inline void rgb24tobgr24_mmxext(const uint8_t *src, uint8_t *dst, int src
}
}
-static inline void yuvPlanartoyuy2_mmxext(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
- int width, int height,
- int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
+static inline void yuvPlanartoyuy2_line_mmxext(uint8_t *dst, const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
+ x86_reg chromWidth)
+{
+ __asm__ volatile(
+ "xor %%"FF_REG_a", %%"FF_REG_a" \n\t"
+ ".p2align 4 \n\t"
+ "1: \n\t"
+ PREFETCH" 32(%1, %%"FF_REG_a", 2) \n\t"
+ PREFETCH" 32(%2, %%"FF_REG_a") \n\t"
+ PREFETCH" 32(%3, %%"FF_REG_a") \n\t"
+ "movq (%2, %%"FF_REG_a"), %%mm0 \n\t" // U(0)
+ "movq %%mm0, %%mm2 \n\t" // U(0)
+ "movq (%3, %%"FF_REG_a"), %%mm1 \n\t" // V(0)
+ "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
+ "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
+
+ "movq (%1, %%"FF_REG_a",2), %%mm3 \n\t" // Y(0)
+ "movq 8(%1, %%"FF_REG_a",2), %%mm5 \n\t" // Y(8)
+ "movq %%mm3, %%mm4 \n\t" // Y(0)
+ "movq %%mm5, %%mm6 \n\t" // Y(8)
+ "punpcklbw %%mm0, %%mm3 \n\t" // YUYV YUYV(0)
+ "punpckhbw %%mm0, %%mm4 \n\t" // YUYV YUYV(4)
+ "punpcklbw %%mm2, %%mm5 \n\t" // YUYV YUYV(8)
+ "punpckhbw %%mm2, %%mm6 \n\t" // YUYV YUYV(12)
+
+ MOVNTQ" %%mm3, (%0, %%"FF_REG_a", 4) \n\t"
+ MOVNTQ" %%mm4, 8(%0, %%"FF_REG_a", 4) \n\t"
+ MOVNTQ" %%mm5, 16(%0, %%"FF_REG_a", 4) \n\t"
+ MOVNTQ" %%mm6, 24(%0, %%"FF_REG_a", 4) \n\t"
+
+ "add $8, %%"FF_REG_a" \n\t"
+ "cmp %4, %%"FF_REG_a" \n\t"
+ " jb 1b \n\t"
+ ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
+ : "%"FF_REG_a
+ );
+}
+
+static inline void yuvPlanartouyvy_line_mmxext(uint8_t *dst, const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc,
+ x86_reg chromWidth)
+{
+ __asm__ volatile(
+ "xor %%"FF_REG_a", %%"FF_REG_a" \n\t"
+ ".p2align 4 \n\t"
+ "1: \n\t"
+ PREFETCH" 32(%1, %%"FF_REG_a", 2) \n\t"
+ PREFETCH" 32(%2, %%"FF_REG_a") \n\t"
+ PREFETCH" 32(%3, %%"FF_REG_a") \n\t"
+ "movq (%2, %%"FF_REG_a"), %%mm0 \n\t" // U(0)
+ "movq %%mm0, %%mm2 \n\t" // U(0)
+ "movq (%3, %%"FF_REG_a"), %%mm1 \n\t" // V(0)
+ "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
+ "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
+
+ "movq (%1, %%"FF_REG_a",2), %%mm3 \n\t" // Y(0)
+ "movq 8(%1, %%"FF_REG_a",2), %%mm5 \n\t" // Y(8)
+ "movq %%mm0, %%mm4 \n\t" // Y(0)
+ "movq %%mm2, %%mm6 \n\t" // Y(8)
+ "punpcklbw %%mm3, %%mm0 \n\t" // YUYV YUYV(0)
+ "punpckhbw %%mm3, %%mm4 \n\t" // YUYV YUYV(4)
+ "punpcklbw %%mm5, %%mm2 \n\t" // YUYV YUYV(8)
+ "punpckhbw %%mm5, %%mm6 \n\t" // YUYV YUYV(12)
+
+ MOVNTQ" %%mm0, (%0, %%"FF_REG_a", 4) \n\t"
+ MOVNTQ" %%mm4, 8(%0, %%"FF_REG_a", 4) \n\t"
+ MOVNTQ" %%mm2, 16(%0, %%"FF_REG_a", 4) \n\t"
+ MOVNTQ" %%mm6, 24(%0, %%"FF_REG_a", 4) \n\t"
+
+ "add $8, %%"FF_REG_a" \n\t"
+ "cmp %4, %%"FF_REG_a" \n\t"
+ " jb 1b \n\t"
+ ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
+ : "%"FF_REG_a
+ );
+}
+
+static inline void yuvPlanartopacked_mmxext(void (*pack_line)(uint8_t *, const uint8_t *, const uint8_t *, const uint8_t *, x86_reg),
+ int luma_first,
+ const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
+ int width, int height,
+ int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
{
const x86_reg chromWidth= width>>1;
for (int y = 0; y < height; y++) {
//FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
- __asm__ volatile(
- "xor %%"FF_REG_a", %%"FF_REG_a" \n\t"
- ".p2align 4 \n\t"
- "1: \n\t"
- PREFETCH" 32(%1, %%"FF_REG_a", 2) \n\t"
- PREFETCH" 32(%2, %%"FF_REG_a") \n\t"
- PREFETCH" 32(%3, %%"FF_REG_a") \n\t"
- "movq (%2, %%"FF_REG_a"), %%mm0 \n\t" // U(0)
- "movq %%mm0, %%mm2 \n\t" // U(0)
- "movq (%3, %%"FF_REG_a"), %%mm1 \n\t" // V(0)
- "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
- "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
-
- "movq (%1, %%"FF_REG_a",2), %%mm3 \n\t" // Y(0)
- "movq 8(%1, %%"FF_REG_a",2), %%mm5 \n\t" // Y(8)
- "movq %%mm3, %%mm4 \n\t" // Y(0)
- "movq %%mm5, %%mm6 \n\t" // Y(8)
- "punpcklbw %%mm0, %%mm3 \n\t" // YUYV YUYV(0)
- "punpckhbw %%mm0, %%mm4 \n\t" // YUYV YUYV(4)
- "punpcklbw %%mm2, %%mm5 \n\t" // YUYV YUYV(8)
- "punpckhbw %%mm2, %%mm6 \n\t" // YUYV YUYV(12)
-
- MOVNTQ" %%mm3, (%0, %%"FF_REG_a", 4) \n\t"
- MOVNTQ" %%mm4, 8(%0, %%"FF_REG_a", 4) \n\t"
- MOVNTQ" %%mm5, 16(%0, %%"FF_REG_a", 4) \n\t"
- MOVNTQ" %%mm6, 24(%0, %%"FF_REG_a", 4) \n\t"
-
- "add $8, %%"FF_REG_a" \n\t"
- "cmp %4, %%"FF_REG_a" \n\t"
- " jb 1b \n\t"
- ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
- : "%"FF_REG_a
- );
+ if (chromWidth < 8) {
+ ff_pack_yuv422_line(dst, ysrc, usrc, vsrc, 0, chromWidth, luma_first);
+ } else {
+ pack_line(dst, ysrc, usrc, vsrc, chromWidth & ~7);
+ if (chromWidth & 7) {
+ x86_reg i = chromWidth - 8;
+ pack_line(dst + 4 * i, ysrc + 2 * i, usrc + i, vsrc + i, 8);
+ }
+ }
if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
usrc += chromStride;
vsrc += chromStride;
@@ -1176,100 +1230,34 @@ static inline void yuvPlanartoyuy2_mmxext(const uint8_t *ysrc, const uint8_t *us
:::"memory");
}
-/**
- * Height should be a multiple of 2 and width should be a multiple of 16.
- * (If this is a problem for anyone then tell me, and I will fix it.)
- */
static inline void yv12toyuy2_mmxext(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
int width, int height,
int lumStride, int chromStride, int dstStride)
{
//FIXME interpolate chroma
- yuvPlanartoyuy2_mmxext(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
-}
-
-static inline void yuvPlanartouyvy_mmxext(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
- int width, int height,
- int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
-{
- const x86_reg chromWidth= width>>1;
- for (int y = 0; y < height; y++) {
- //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
- __asm__ volatile(
- "xor %%"FF_REG_a", %%"FF_REG_a" \n\t"
- ".p2align 4 \n\t"
- "1: \n\t"
- PREFETCH" 32(%1, %%"FF_REG_a", 2) \n\t"
- PREFETCH" 32(%2, %%"FF_REG_a") \n\t"
- PREFETCH" 32(%3, %%"FF_REG_a") \n\t"
- "movq (%2, %%"FF_REG_a"), %%mm0 \n\t" // U(0)
- "movq %%mm0, %%mm2 \n\t" // U(0)
- "movq (%3, %%"FF_REG_a"), %%mm1 \n\t" // V(0)
- "punpcklbw %%mm1, %%mm0 \n\t" // UVUV UVUV(0)
- "punpckhbw %%mm1, %%mm2 \n\t" // UVUV UVUV(8)
-
- "movq (%1, %%"FF_REG_a",2), %%mm3 \n\t" // Y(0)
- "movq 8(%1, %%"FF_REG_a",2), %%mm5 \n\t" // Y(8)
- "movq %%mm0, %%mm4 \n\t" // Y(0)
- "movq %%mm2, %%mm6 \n\t" // Y(8)
- "punpcklbw %%mm3, %%mm0 \n\t" // YUYV YUYV(0)
- "punpckhbw %%mm3, %%mm4 \n\t" // YUYV YUYV(4)
- "punpcklbw %%mm5, %%mm2 \n\t" // YUYV YUYV(8)
- "punpckhbw %%mm5, %%mm6 \n\t" // YUYV YUYV(12)
-
- MOVNTQ" %%mm0, (%0, %%"FF_REG_a", 4) \n\t"
- MOVNTQ" %%mm4, 8(%0, %%"FF_REG_a", 4) \n\t"
- MOVNTQ" %%mm2, 16(%0, %%"FF_REG_a", 4) \n\t"
- MOVNTQ" %%mm6, 24(%0, %%"FF_REG_a", 4) \n\t"
-
- "add $8, %%"FF_REG_a" \n\t"
- "cmp %4, %%"FF_REG_a" \n\t"
- " jb 1b \n\t"
- ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
- : "%"FF_REG_a
- );
- if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
- usrc += chromStride;
- vsrc += chromStride;
- }
- ysrc += lumStride;
- dst += dstStride;
- }
- __asm__(EMMS" \n\t"
- SFENCE" \n\t"
- :::"memory");
+ yuvPlanartopacked_mmxext(yuvPlanartoyuy2_line_mmxext, 1, ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
}
-/**
- * Height should be a multiple of 2 and width should be a multiple of 16
- * (If this is a problem for anyone then tell me, and I will fix it.)
- */
static inline void yv12touyvy_mmxext(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
int width, int height,
int lumStride, int chromStride, int dstStride)
{
//FIXME interpolate chroma
- yuvPlanartouyvy_mmxext(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
+ yuvPlanartopacked_mmxext(yuvPlanartouyvy_line_mmxext, 0, ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
}
-/**
- * Width should be a multiple of 16.
- */
static inline void yuv422ptouyvy_mmxext(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
int width, int height,
int lumStride, int chromStride, int dstStride)
{
- yuvPlanartouyvy_mmxext(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
+ yuvPlanartopacked_mmxext(yuvPlanartouyvy_line_mmxext, 0, ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
}
-/**
- * Width should be a multiple of 16.
- */
static inline void yuv422ptoyuy2_mmxext(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
int width, int height,
int lumStride, int chromStride, int dstStride)
{
- yuvPlanartoyuy2_mmxext(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
+ yuvPlanartopacked_mmxext(yuvPlanartoyuy2_line_mmxext, 1, ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
}
static inline void planar2x_mmxext(const uint8_t *src, uint8_t *dst, int srcWidth, int srcHeight, int srcStride, int dstStride)