Commit ea4162f223 for ffmpeg
commit ea4162f223e5068c13210a6cc7a84c7d84f2ba3a
Author: Gyula Sebok <gyula.sbk@gmail.com>
Date: Sat Oct 31 19:19:37 2026 +0100
avcodec/aarch64: Add NEON llviddsp left and gradient prediction
Like the x86 versions, these may read and write past the end, here by
up to 15 bytes. The main loops process eight vectors per iteration;
with four, the in-order Cortex-A510 stalls on the latency of the prefix
sums, and add_left_pred_int16 was only 1.25x faster than C there.
checkasm --bench (CPU cycles via perf), seed 12345 (width 983), on a
Dimensity 9000:
Cortex-X2 Cortex-A710 Cortex-A510
add_gradient_pred_c: 2138.4 2183.6 4450.3
add_gradient_pred_neon: 225.7 (9.48x) 380.3 (5.74x) 711.6 (6.25x)
add_left_pred_int16_c: 1926.4 1941.4 2112.0
add_left_pred_int16_neon: 370.6 (5.20x) 630.6 (3.08x) 1058.6 (2.00x)
add_left_pred_rnd_acc_c: 1056.5 1029.6 1743.4
add_left_pred_rnd_acc_neon: 201.9 (5.23x) 347.5 (2.96x) 585.2 (2.98x)
add_left_pred_zero_c: 1056.0 1029.2 1742.3
add_left_pred_zero_neon: 201.8 (5.23x) 347.5 (2.96x) 585.2 (2.98x)
and on a Google Tensor (Pixel 6a):
Cortex-X1 Cortex-A76 Cortex-A55
add_gradient_pred_c: 2137.4 2848.2 6892.2
add_gradient_pred_neon: 238.2 (8.97x) 408.2 (6.98x) 1117.5 (6.17x)
add_left_pred_int16_c: 1935.0 2177.2 3950.8
add_left_pred_int16_neon: 398.0 (4.86x) 664.4 (3.28x) 1690.0 (2.34x)
add_left_pred_rnd_acc_c: 1059.7 1533.9 2967.1
add_left_pred_rnd_acc_neon: 214.5 (4.94x) 378.2 (4.06x) 925.4 (3.21x)
add_left_pred_zero_c: 1059.8 1533.9 2967.1
add_left_pred_zero_neon: 214.5 (4.94x) 378.2 (4.06x) 925.4 (3.21x)
Signed-off-by: Gyula Sebok <gyula.sbk@gmail.com>
diff --git a/libavcodec/aarch64/Makefile b/libavcodec/aarch64/Makefile
index 551c45b055..e560a8c312 100644
--- a/libavcodec/aarch64/Makefile
+++ b/libavcodec/aarch64/Makefile
@@ -9,6 +9,7 @@ OBJS-$(CONFIG_H264PRED) += aarch64/h264pred_init.o
OBJS-$(CONFIG_H264QPEL) += aarch64/h264qpel_init_aarch64.o
OBJS-$(CONFIG_HPELDSP) += aarch64/hpeldsp_init_aarch64.o
OBJS-$(CONFIG_IDCTDSP) += aarch64/idctdsp_init_aarch64.o
+OBJS-$(CONFIG_LLVIDDSP) += aarch64/lossless_videodsp_init.o
OBJS-$(CONFIG_ME_CMP) += aarch64/me_cmp_init_aarch64.o
OBJS-$(CONFIG_MPEGAUDIODSP) += aarch64/mpegaudiodsp_init.o
OBJS-$(CONFIG_MPEGVIDEOENCDSP) += aarch64/mpegvideoencdsp_init.o
@@ -60,6 +61,7 @@ NEON-OBJS-$(CONFIG_HPELDSP) += aarch64/hpeldsp_neon.o
NEON-OBJS-$(CONFIG_HUFFYUVDSP) += aarch64/huffyuvdsp_neon.o
NEON-OBJS-$(CONFIG_IDCTDSP) += aarch64/idctdsp_neon.o \
aarch64/simple_idct_neon.o
+NEON-OBJS-$(CONFIG_LLVIDDSP) += aarch64/lossless_videodsp_neon.o
NEON-OBJS-$(CONFIG_ME_CMP) += aarch64/me_cmp_neon.o
NEON-OBJS-$(CONFIG_MPEGAUDIODSP) += aarch64/mpegaudiodsp_neon.o
NEON-OBJS-$(CONFIG_MPEGVIDEOENCDSP) += aarch64/mpegvideoencdsp_neon.o
diff --git a/libavcodec/aarch64/lossless_videodsp_init.c b/libavcodec/aarch64/lossless_videodsp_init.c
new file mode 100644
index 0000000000..2f5149795a
--- /dev/null
+++ b/libavcodec/aarch64/lossless_videodsp_init.c
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2026 Gyula Sebok <gyula.sbk@gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/attributes.h"
+#include "libavutil/cpu.h"
+#include "libavutil/aarch64/cpu.h"
+#include "libavcodec/lossless_videodsp.h"
+
+int ff_add_left_pred_neon(uint8_t *dst, const uint8_t *src,
+ ptrdiff_t w, int acc);
+int ff_add_left_pred_int16_neon(uint16_t *dst, const uint16_t *src,
+ unsigned mask, ptrdiff_t w, unsigned acc);
+void ff_add_gradient_pred_neon(uint8_t *src, const ptrdiff_t stride,
+ const ptrdiff_t width);
+
+av_cold void ff_llviddsp_init_aarch64(LLVidDSPContext *c)
+{
+ int cpu_flags = av_get_cpu_flags();
+
+ if (have_neon(cpu_flags)) {
+ c->add_left_pred = ff_add_left_pred_neon;
+ c->add_left_pred_int16 = ff_add_left_pred_int16_neon;
+ c->add_gradient_pred = ff_add_gradient_pred_neon;
+ }
+}
diff --git a/libavcodec/aarch64/lossless_videodsp_neon.S b/libavcodec/aarch64/lossless_videodsp_neon.S
new file mode 100644
index 0000000000..9bccb9e188
--- /dev/null
+++ b/libavcodec/aarch64/lossless_videodsp_neon.S
@@ -0,0 +1,268 @@
+/*
+ * Copyright (c) 2026 Gyula Sebok <gyula.sbk@gmail.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/aarch64/asm.S"
+
+// v31 = 0, clobbers v16-v23
+.macro prefix_step sz, shift, nregs
+ ext v16.16b, v31.16b, v0.16b, #16 - \shift
+.if \nregs >= 4
+ ext v17.16b, v31.16b, v1.16b, #16 - \shift
+ ext v18.16b, v31.16b, v2.16b, #16 - \shift
+ ext v19.16b, v31.16b, v3.16b, #16 - \shift
+.endif
+.if \nregs == 8
+ ext v20.16b, v31.16b, v4.16b, #16 - \shift
+ ext v21.16b, v31.16b, v5.16b, #16 - \shift
+ ext v22.16b, v31.16b, v6.16b, #16 - \shift
+ ext v23.16b, v31.16b, v7.16b, #16 - \shift
+.endif
+ add v0\sz, v0\sz, v16\sz
+.if \nregs >= 4
+ add v1\sz, v1\sz, v17\sz
+ add v2\sz, v2\sz, v18\sz
+ add v3\sz, v3\sz, v19\sz
+.endif
+.if \nregs == 8
+ add v4\sz, v4\sz, v20\sz
+ add v5\sz, v5\sz, v21\sz
+ add v6\sz, v6\sz, v22\sz
+ add v7\sz, v7\sz, v23\sz
+.endif
+.endm
+
+.macro prefix_sum_8 nregs
+ prefix_step .16b, 1, \nregs
+ prefix_step .16b, 2, \nregs
+ prefix_step .16b, 4, \nregs
+ prefix_step .16b, 8, \nregs
+.endm
+
+.macro prefix_sum_16 nregs
+ prefix_step .8h, 2, \nregs
+ prefix_step .8h, 4, \nregs
+ prefix_step .8h, 8, \nregs
+.endm
+
+// v0-v3 or v0-v7 += v30 + the last elements of the preceding vectors,
+// v30 = the new running sum. Only v0 and v1 get v30 directly, the others
+// get it through the broadcasts of v1 and v3. mask=1 also ands the
+// results with v29. Clobbers v16-v22.
+.macro add_carry sz, last, nregs, mask=0
+ add v1\sz, v1\sz, v30\sz
+ dup v16\sz, v0\last
+ dup v18\sz, v2\last
+.if \nregs == 8
+ dup v20\sz, v4\last
+ dup v22\sz, v6\last
+.endif
+ add v0\sz, v0\sz, v30\sz
+ add v1\sz, v1\sz, v16\sz
+ add v3\sz, v3\sz, v18\sz
+.if \nregs == 8
+ add v5\sz, v5\sz, v20\sz
+ add v7\sz, v7\sz, v22\sz
+.endif
+.if \mask
+ and v0.16b, v0.16b, v29.16b
+.endif
+ dup v17\sz, v1\last
+.if \nregs == 8
+ dup v21\sz, v5\last
+.endif
+.if \mask
+ and v1.16b, v1.16b, v29.16b
+.endif
+ add v2\sz, v2\sz, v17\sz
+ add v3\sz, v3\sz, v17\sz
+.if \nregs == 4
+ dup v30\sz, v3\last
+.else
+ add v6\sz, v6\sz, v21\sz
+ add v7\sz, v7\sz, v21\sz
+.if \mask
+ and v2.16b, v2.16b, v29.16b
+.endif
+ dup v19\sz, v3\last
+.if \mask
+ and v3.16b, v3.16b, v29.16b
+.endif
+ add v4\sz, v4\sz, v19\sz
+ add v5\sz, v5\sz, v19\sz
+ add v6\sz, v6\sz, v19\sz
+ add v7\sz, v7\sz, v19\sz
+ dup v30\sz, v7\last
+.endif
+.if \mask
+.if \nregs == 4
+ and v2.16b, v2.16b, v29.16b
+ and v3.16b, v3.16b, v29.16b
+.else
+ and v4.16b, v4.16b, v29.16b
+ and v5.16b, v5.16b, v29.16b
+ and v6.16b, v6.16b, v29.16b
+ and v7.16b, v7.16b, v29.16b
+.endif
+.endif
+.endm
+
+// int ff_add_left_pred_neon(uint8_t *dst, const uint8_t *src,
+// ptrdiff_t w, int acc)
+function ff_add_left_pred_neon, export=1
+ add x4, x0, x2 // dst + w
+ movi v31.16b, #0
+ dup v30.16b, w3 // running sum
+ subs x2, x2, #128
+ b.lt 2f
+1:
+ ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #64
+ ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x1], #64
+ prefix_sum_8 8
+ add_carry .16b, .b[15], 8
+ subs x2, x2, #128
+ st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64
+ st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #64
+ b.ge 1b
+2:
+ tbz x2, #6, 3f // the low 7 bits are w % 128
+ ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x1], #64
+ prefix_sum_8 4
+ add_carry .16b, .b[15], 4
+ st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64
+3:
+ ands x2, x2, #63
+ b.eq 5f
+4:
+ ld1 {v0.16b}, [x1], #16
+ prefix_sum_8 1
+ add v0.16b, v0.16b, v30.16b
+ subs x2, x2, #16
+ dup v30.16b, v0.b[15]
+ st1 {v0.16b}, [x0], #16
+ b.gt 4b
+ ldurb w0, [x4, #-1] // v30 can be past dst[w - 1]
+ ret
+5:
+ umov w0, v30.b[0]
+ ret
+endfunc
+
+// int ff_add_left_pred_int16_neon(uint16_t *dst, const uint16_t *src,
+// unsigned mask, ptrdiff_t w, unsigned acc)
+function ff_add_left_pred_int16_neon, export=1
+ add x5, x0, x3, lsl #1 // dst + w
+ movi v31.16b, #0
+ dup v29.8h, w2 // mask, (1 << bits) - 1
+ dup v30.8h, w4 // running sum
+ subs x3, x3, #64
+ b.lt 2f
+1:
+ ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], #64
+ ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x1], #64
+ prefix_sum_16 8
+ add_carry .8h, .h[7], 8, mask=1
+ subs x3, x3, #64
+ st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
+ st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x0], #64
+ b.ge 1b
+2:
+ tbz x3, #5, 3f // the low 6 bits are w % 64
+ ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x1], #64
+ prefix_sum_16 4
+ add_carry .8h, .h[7], 4, mask=1
+ st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x0], #64
+3:
+ ands x3, x3, #31
+ b.eq 5f
+4:
+ ld1 {v0.8h}, [x1], #16
+ prefix_sum_16 1
+ add v0.8h, v0.8h, v30.8h
+ subs x3, x3, #8
+ dup v30.8h, v0.h[7]
+ and v0.16b, v0.16b, v29.16b
+ st1 {v0.8h}, [x0], #16
+ b.gt 4b
+ ldurh w0, [x5, #-2] // v30 can be past dst[w - 1]
+ ret
+5:
+ umov w0, v30.h[0]
+ and w0, w0, w2 // v30 is not masked
+ ret
+endfunc
+
+// \r0-\r3 += top, clobbers v16-v19
+.macro add_top4 r0, r1, r2, r3
+ ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x3], #64
+ add \r0\().16b, \r0\().16b, v16.16b
+ add \r1\().16b, \r1\().16b, v17.16b
+ add \r2\().16b, \r2\().16b, v18.16b
+ add \r3\().16b, \r3\().16b, v19.16b
+.endm
+
+// void ff_add_gradient_pred_neon(uint8_t *src, const ptrdiff_t stride,
+// const ptrdiff_t width)
+// For residuals D and reconstructed top row T, the gradient recurrence is
+// dst[x] = left - T[-1] + sum(D[0..x]) + T[x] (mod 256). Keep the
+// adjusted residual sum in v30, adding T only after the prefix scan.
+function ff_add_gradient_pred_neon, export=1
+ sub x3, x0, x1 // top
+ mov x5, x0 // read pointer
+ ldurb w6, [x0, #-1]
+ ldurb w7, [x3, #-1]
+ sub w6, w6, w7
+ movi v31.16b, #0
+ dup v30.16b, w6 // left - top left
+ subs x2, x2, #128
+ b.lt 2f
+1:
+ ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x5], #64
+ ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x5], #64
+ prefix_sum_8 8
+ add_carry .16b, .b[15], 8
+ add_top4 v0, v1, v2, v3
+ add_top4 v4, v5, v6, v7
+ subs x2, x2, #128
+ st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64
+ st1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x0], #64
+ b.ge 1b
+2:
+ tbz x2, #6, 3f
+ ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x5], #64
+ prefix_sum_8 4
+ add_carry .16b, .b[15], 4
+ add_top4 v0, v1, v2, v3
+ st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x0], #64
+3:
+ ands x2, x2, #63
+ b.eq 9f
+4:
+ ld1 {v0.16b}, [x5], #16
+ prefix_sum_8 1
+ add v0.16b, v0.16b, v30.16b
+ subs x2, x2, #16
+ dup v30.16b, v0.b[15]
+ ld1 {v16.16b}, [x3], #16
+ add v0.16b, v0.16b, v16.16b
+ st1 {v0.16b}, [x0], #16
+ b.gt 4b
+9:
+ ret
+endfunc
diff --git a/libavcodec/lossless_videodsp.c b/libavcodec/lossless_videodsp.c
index bf3a3da90d..ffb6f8a555 100644
--- a/libavcodec/lossless_videodsp.c
+++ b/libavcodec/lossless_videodsp.c
@@ -120,7 +120,9 @@ av_cold void ff_llviddsp_init(LLVidDSPContext *c)
c->add_left_pred_int16 = add_left_pred_int16_c;
c->add_gradient_pred = add_gradient_pred_c;
-#if ARCH_PPC
+#if ARCH_AARCH64
+ ff_llviddsp_init_aarch64(c);
+#elif ARCH_PPC
ff_llviddsp_init_ppc(c);
#elif ARCH_RISCV
ff_llviddsp_init_riscv(c);
diff --git a/libavcodec/lossless_videodsp.h b/libavcodec/lossless_videodsp.h
index 5309ce4be7..f28a3824e6 100644
--- a/libavcodec/lossless_videodsp.h
+++ b/libavcodec/lossless_videodsp.h
@@ -40,6 +40,7 @@ typedef struct LLVidDSPContext {
} LLVidDSPContext;
void ff_llviddsp_init(LLVidDSPContext *llviddsp);
+void ff_llviddsp_init_aarch64(LLVidDSPContext *llviddsp);
void ff_llviddsp_init_riscv(LLVidDSPContext *llviddsp);
void ff_llviddsp_init_x86(LLVidDSPContext *llviddsp);
void ff_llviddsp_init_ppc(LLVidDSPContext *llviddsp);