Commit e0425ecc02 for aom
commit e0425ecc02018ed18a942d46aa08be4972726d2d
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Nov 14 10:41:48 2026 -0800
aom_dsp,neon: Unroll subtract & sum_squares_2d
Unroll aom_subtract_block_neon for both lowbd and highbd configurations
across 2-row and 4-row iterations. Optimize aom_sum_squares_2d_i16_neon
with 4x4 and 4xN vector unrolls using vcombine and vpadal.
Speedup: -0.92% (1T ARM64) encode time reduction (bit-exact).
TAG=agy
CONV=490488d4-7005-402a-8cc4-c245227f80fe
Change-Id: Ia5834c2b1bbe3b006dfa72f9d7bc94e1c5bbdeb4
diff --git a/aom_dsp/arm/subtract_neon.c b/aom_dsp/arm/subtract_neon.c
index b093e32739..2da31d93f7 100644
--- a/aom_dsp/arm/subtract_neon.c
+++ b/aom_dsp/arm/subtract_neon.c
@@ -15,6 +15,7 @@
#include "config/aom_dsp_rtcd.h"
#include "aom/aom_integer.h"
+#include "aom_dsp/arm/mem_neon.h"
#include "aom_ports/mem.h"
void aom_subtract_block_neon(int rows, int cols, int16_t *diff,
@@ -23,34 +24,112 @@ void aom_subtract_block_neon(int rows, int cols, int16_t *diff,
ptrdiff_t pred_stride) {
if (cols > 16) {
int r = rows;
- do {
- int c = 0;
+ if (cols == 32) {
+ while (r >= 2) {
+ const uint8x16_t s0_0 = vld1q_u8(&src[0]);
+ const uint8x16_t s0_1 = vld1q_u8(&src[16]);
+ const uint8x16_t p0_0 = vld1q_u8(&pred[0]);
+ const uint8x16_t p0_1 = vld1q_u8(&pred[16]);
+ const uint8x16_t s1_0 = vld1q_u8(&src[src_stride + 0]);
+ const uint8x16_t s1_1 = vld1q_u8(&src[src_stride + 16]);
+ const uint8x16_t p1_0 = vld1q_u8(&pred[pred_stride + 0]);
+ const uint8x16_t p1_1 = vld1q_u8(&pred[pred_stride + 16]);
+ const int16x8_t d00 = vreinterpretq_s16_u16(
+ vsubl_u8(vget_low_u8(s0_0), vget_low_u8(p0_0)));
+ const int16x8_t d01 = vreinterpretq_s16_u16(
+ vsubl_u8(vget_high_u8(s0_0), vget_high_u8(p0_0)));
+ const int16x8_t d02 = vreinterpretq_s16_u16(
+ vsubl_u8(vget_low_u8(s0_1), vget_low_u8(p0_1)));
+ const int16x8_t d03 = vreinterpretq_s16_u16(
+ vsubl_u8(vget_high_u8(s0_1), vget_high_u8(p0_1)));
+ int16_t *diff1 = diff + diff_stride;
+ const int16x8_t d10 = vreinterpretq_s16_u16(
+ vsubl_u8(vget_low_u8(s1_0), vget_low_u8(p1_0)));
+ const int16x8_t d11 = vreinterpretq_s16_u16(
+ vsubl_u8(vget_high_u8(s1_0), vget_high_u8(p1_0)));
+ const int16x8_t d12 = vreinterpretq_s16_u16(
+ vsubl_u8(vget_low_u8(s1_1), vget_low_u8(p1_1)));
+ const int16x8_t d13 = vreinterpretq_s16_u16(
+ vsubl_u8(vget_high_u8(s1_1), vget_high_u8(p1_1)));
+ vst1q_s16(&diff[0], d00);
+ vst1q_s16(&diff[8], d01);
+ vst1q_s16(&diff[16], d02);
+ vst1q_s16(&diff[24], d03);
+ vst1q_s16(&diff1[0], d10);
+ vst1q_s16(&diff1[8], d11);
+ vst1q_s16(&diff1[16], d12);
+ vst1q_s16(&diff1[24], d13);
+ diff += diff_stride << 1;
+ pred += pred_stride << 1;
+ src += src_stride << 1;
+ r -= 2;
+ }
+ if (r != 0) {
+ const uint8x16_t s0 = vld1q_u8(&src[0]);
+ const uint8x16_t s1 = vld1q_u8(&src[16]);
+ const uint8x16_t p0 = vld1q_u8(&pred[0]);
+ const uint8x16_t p1 = vld1q_u8(&pred[16]);
+ vst1q_s16(&diff[0], vreinterpretq_s16_u16(
+ vsubl_u8(vget_low_u8(s0), vget_low_u8(p0))));
+ vst1q_s16(&diff[8], vreinterpretq_s16_u16(
+ vsubl_u8(vget_high_u8(s0), vget_high_u8(p0))));
+ vst1q_s16(&diff[16], vreinterpretq_s16_u16(
+ vsubl_u8(vget_low_u8(s1), vget_low_u8(p1))));
+ vst1q_s16(&diff[24], vreinterpretq_s16_u16(
+ vsubl_u8(vget_high_u8(s1), vget_high_u8(p1))));
+ }
+ } else {
do {
- const uint8x16_t v_src_00 = vld1q_u8(&src[c + 0]);
- const uint8x16_t v_src_16 = vld1q_u8(&src[c + 16]);
- const uint8x16_t v_pred_00 = vld1q_u8(&pred[c + 0]);
- const uint8x16_t v_pred_16 = vld1q_u8(&pred[c + 16]);
- const uint16x8_t v_diff_lo_00 =
- vsubl_u8(vget_low_u8(v_src_00), vget_low_u8(v_pred_00));
- const uint16x8_t v_diff_hi_00 =
- vsubl_u8(vget_high_u8(v_src_00), vget_high_u8(v_pred_00));
- const uint16x8_t v_diff_lo_16 =
- vsubl_u8(vget_low_u8(v_src_16), vget_low_u8(v_pred_16));
- const uint16x8_t v_diff_hi_16 =
- vsubl_u8(vget_high_u8(v_src_16), vget_high_u8(v_pred_16));
- vst1q_s16(&diff[c + 0], vreinterpretq_s16_u16(v_diff_lo_00));
- vst1q_s16(&diff[c + 8], vreinterpretq_s16_u16(v_diff_hi_00));
- vst1q_s16(&diff[c + 16], vreinterpretq_s16_u16(v_diff_lo_16));
- vst1q_s16(&diff[c + 24], vreinterpretq_s16_u16(v_diff_hi_16));
- c += 32;
- } while (c < cols);
- diff += diff_stride;
- pred += pred_stride;
- src += src_stride;
- } while (--r != 0);
+ int c = 0;
+ do {
+ const uint8x16_t v_src_00 = vld1q_u8(&src[c + 0]);
+ const uint8x16_t v_src_16 = vld1q_u8(&src[c + 16]);
+ const uint8x16_t v_pred_00 = vld1q_u8(&pred[c + 0]);
+ const uint8x16_t v_pred_16 = vld1q_u8(&pred[c + 16]);
+ const uint16x8_t v_diff_lo_00 =
+ vsubl_u8(vget_low_u8(v_src_00), vget_low_u8(v_pred_00));
+ const uint16x8_t v_diff_hi_00 =
+ vsubl_u8(vget_high_u8(v_src_00), vget_high_u8(v_pred_00));
+ const uint16x8_t v_diff_lo_16 =
+ vsubl_u8(vget_low_u8(v_src_16), vget_low_u8(v_pred_16));
+ const uint16x8_t v_diff_hi_16 =
+ vsubl_u8(vget_high_u8(v_src_16), vget_high_u8(v_pred_16));
+ vst1q_s16(&diff[c + 0], vreinterpretq_s16_u16(v_diff_lo_00));
+ vst1q_s16(&diff[c + 8], vreinterpretq_s16_u16(v_diff_hi_00));
+ vst1q_s16(&diff[c + 16], vreinterpretq_s16_u16(v_diff_lo_16));
+ vst1q_s16(&diff[c + 24], vreinterpretq_s16_u16(v_diff_hi_16));
+ c += 32;
+ } while (c < cols);
+ diff += diff_stride;
+ pred += pred_stride;
+ src += src_stride;
+ } while (--r != 0);
+ }
} else if (cols > 8) {
int r = rows;
- do {
+ while (r >= 2) {
+ const uint8x16_t v_src0 = vld1q_u8(&src[0]);
+ const uint8x16_t v_pred0 = vld1q_u8(&pred[0]);
+ const uint8x16_t v_src1 = vld1q_u8(&src[src_stride]);
+ const uint8x16_t v_pred1 = vld1q_u8(&pred[pred_stride]);
+ const uint16x8_t v_diff0_lo =
+ vsubl_u8(vget_low_u8(v_src0), vget_low_u8(v_pred0));
+ const uint16x8_t v_diff0_hi =
+ vsubl_u8(vget_high_u8(v_src0), vget_high_u8(v_pred0));
+ const uint16x8_t v_diff1_lo =
+ vsubl_u8(vget_low_u8(v_src1), vget_low_u8(v_pred1));
+ const uint16x8_t v_diff1_hi =
+ vsubl_u8(vget_high_u8(v_src1), vget_high_u8(v_pred1));
+ vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff0_lo));
+ vst1q_s16(&diff[8], vreinterpretq_s16_u16(v_diff0_hi));
+ vst1q_s16(&diff[diff_stride + 0], vreinterpretq_s16_u16(v_diff1_lo));
+ vst1q_s16(&diff[diff_stride + 8], vreinterpretq_s16_u16(v_diff1_hi));
+ diff += diff_stride << 1;
+ pred += pred_stride << 1;
+ src += src_stride << 1;
+ r -= 2;
+ }
+ if (r != 0) {
const uint8x16_t v_src = vld1q_u8(&src[0]);
const uint8x16_t v_pred = vld1q_u8(&pred[0]);
const uint16x8_t v_diff_lo =
@@ -59,21 +138,73 @@ void aom_subtract_block_neon(int rows, int cols, int16_t *diff,
vsubl_u8(vget_high_u8(v_src), vget_high_u8(v_pred));
vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff_lo));
vst1q_s16(&diff[8], vreinterpretq_s16_u16(v_diff_hi));
- diff += diff_stride;
- pred += pred_stride;
- src += src_stride;
- } while (--r != 0);
+ }
} else if (cols > 4) {
int r = rows;
- do {
- const uint8x8_t v_src = vld1_u8(&src[0]);
- const uint8x8_t v_pred = vld1_u8(&pred[0]);
- const uint16x8_t v_diff = vsubl_u8(v_src, v_pred);
- vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff));
- diff += diff_stride;
- pred += pred_stride;
- src += src_stride;
- } while (--r != 0);
+ while (r >= 2) {
+ const uint8x8_t v_src0 = vld1_u8(&src[0]);
+ const uint8x8_t v_src1 = vld1_u8(&src[src_stride]);
+ const uint8x8_t v_pred0 = vld1_u8(&pred[0]);
+ const uint8x8_t v_pred1 = vld1_u8(&pred[pred_stride]);
+ const uint16x8_t v_diff0 = vsubl_u8(v_src0, v_pred0);
+ const uint16x8_t v_diff1 = vsubl_u8(v_src1, v_pred1);
+ vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff0));
+ vst1q_s16(&diff[diff_stride], vreinterpretq_s16_u16(v_diff1));
+ diff += diff_stride << 1;
+ pred += pred_stride << 1;
+ src += src_stride << 1;
+ r -= 2;
+ }
+ if (r != 0) {
+ do {
+ const uint8x8_t v_src = vld1_u8(&src[0]);
+ const uint8x8_t v_pred = vld1_u8(&pred[0]);
+ const uint16x8_t v_diff = vsubl_u8(v_src, v_pred);
+ vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff));
+ diff += diff_stride;
+ pred += pred_stride;
+ src += src_stride;
+ } while (--r != 0);
+ }
+ } else if (cols == 4) {
+ int r = rows;
+ if (r >= 4) {
+ do {
+ uint8x8_t v_src_01, v_src_23, v_pred_01, v_pred_23;
+ load_unaligned_u8_4x4(src, src_stride, &v_src_01, &v_src_23);
+ load_unaligned_u8_4x4(pred, pred_stride, &v_pred_01, &v_pred_23);
+ const int16x8_t v_diff_01 =
+ vreinterpretq_s16_u16(vsubl_u8(v_src_01, v_pred_01));
+ const int16x8_t v_diff_23 =
+ vreinterpretq_s16_u16(vsubl_u8(v_src_23, v_pred_23));
+ vst1_s16(&diff[0 * diff_stride], vget_low_s16(v_diff_01));
+ vst1_s16(&diff[1 * diff_stride], vget_high_s16(v_diff_01));
+ vst1_s16(&diff[2 * diff_stride], vget_low_s16(v_diff_23));
+ vst1_s16(&diff[3 * diff_stride], vget_high_s16(v_diff_23));
+ diff += diff_stride << 2;
+ pred += pred_stride << 2;
+ src += src_stride << 2;
+ r -= 4;
+ } while (r >= 4);
+ }
+ while (r >= 2) {
+ const uint8x8_t v_src_01 = load_unaligned_u8_4x2(src, src_stride);
+ const uint8x8_t v_pred_01 = load_unaligned_u8_4x2(pred, pred_stride);
+ const int16x8_t v_diff_01 =
+ vreinterpretq_s16_u16(vsubl_u8(v_src_01, v_pred_01));
+ vst1_s16(&diff[0 * diff_stride], vget_low_s16(v_diff_01));
+ vst1_s16(&diff[1 * diff_stride], vget_high_s16(v_diff_01));
+ diff += diff_stride << 1;
+ pred += pred_stride << 1;
+ src += src_stride << 1;
+ r -= 2;
+ }
+ if (r != 0) {
+ const uint8x8_t v_src = load_unaligned_u8_4x1(src);
+ const uint8x8_t v_pred = load_unaligned_u8_4x1(pred);
+ const int16x8_t v_diff = vreinterpretq_s16_u16(vsubl_u8(v_src, v_pred));
+ vst1_s16(diff, vget_low_s16(v_diff));
+ }
} else {
int r = rows;
do {
@@ -107,9 +238,17 @@ void aom_highbd_subtract_block_neon(int rows, int cols, int16_t *diff,
const uint16x8_t v_src_08 = vld1q_u16(&src[c + 8]);
const uint16x8_t v_pred_08 = vld1q_u16(&pred[c + 8]);
const uint16x8_t v_diff_08 = vsubq_u16(v_src_08, v_pred_08);
+ const uint16x8_t v_src_16 = vld1q_u16(&src[c + 16]);
+ const uint16x8_t v_pred_16 = vld1q_u16(&pred[c + 16]);
+ const uint16x8_t v_diff_16 = vsubq_u16(v_src_16, v_pred_16);
+ const uint16x8_t v_src_24 = vld1q_u16(&src[c + 24]);
+ const uint16x8_t v_pred_24 = vld1q_u16(&pred[c + 24]);
+ const uint16x8_t v_diff_24 = vsubq_u16(v_src_24, v_pred_24);
vst1q_s16(&diff[c + 0], vreinterpretq_s16_u16(v_diff_00));
vst1q_s16(&diff[c + 8], vreinterpretq_s16_u16(v_diff_08));
- c += 16;
+ vst1q_s16(&diff[c + 16], vreinterpretq_s16_u16(v_diff_16));
+ vst1q_s16(&diff[c + 24], vreinterpretq_s16_u16(v_diff_24));
+ c += 32;
} while (c < cols);
diff += diff_stride;
pred += pred_stride;
@@ -117,19 +256,38 @@ void aom_highbd_subtract_block_neon(int rows, int cols, int16_t *diff,
} while (--r != 0);
} else if (cols > 8) {
int r = rows;
- do {
+ while (r >= 2) {
const uint16x8_t v_src_00 = vld1q_u16(&src[0]);
const uint16x8_t v_pred_00 = vld1q_u16(&pred[0]);
- const uint16x8_t v_diff_00 = vsubq_u16(v_src_00, v_pred_00);
const uint16x8_t v_src_08 = vld1q_u16(&src[8]);
const uint16x8_t v_pred_08 = vld1q_u16(&pred[8]);
- const uint16x8_t v_diff_08 = vsubq_u16(v_src_08, v_pred_08);
- vst1q_s16(&diff[0], vreinterpretq_s16_u16(v_diff_00));
- vst1q_s16(&diff[8], vreinterpretq_s16_u16(v_diff_08));
- diff += diff_stride;
- pred += pred_stride;
- src += src_stride;
- } while (--r != 0);
+ const uint16x8_t v_src_10 = vld1q_u16(&src[src_stride + 0]);
+ const uint16x8_t v_pred_10 = vld1q_u16(&pred[pred_stride + 0]);
+ const uint16x8_t v_src_18 = vld1q_u16(&src[src_stride + 8]);
+ const uint16x8_t v_pred_18 = vld1q_u16(&pred[pred_stride + 8]);
+ vst1q_s16(&diff[0],
+ vreinterpretq_s16_u16(vsubq_u16(v_src_00, v_pred_00)));
+ vst1q_s16(&diff[8],
+ vreinterpretq_s16_u16(vsubq_u16(v_src_08, v_pred_08)));
+ vst1q_s16(&diff[diff_stride + 0],
+ vreinterpretq_s16_u16(vsubq_u16(v_src_10, v_pred_10)));
+ vst1q_s16(&diff[diff_stride + 8],
+ vreinterpretq_s16_u16(vsubq_u16(v_src_18, v_pred_18)));
+ diff += diff_stride << 1;
+ pred += pred_stride << 1;
+ src += src_stride << 1;
+ r -= 2;
+ }
+ if (r != 0) {
+ const uint16x8_t v_src_00 = vld1q_u16(&src[0]);
+ const uint16x8_t v_pred_00 = vld1q_u16(&pred[0]);
+ const uint16x8_t v_src_08 = vld1q_u16(&src[8]);
+ const uint16x8_t v_pred_08 = vld1q_u16(&pred[8]);
+ vst1q_s16(&diff[0],
+ vreinterpretq_s16_u16(vsubq_u16(v_src_00, v_pred_00)));
+ vst1q_s16(&diff[8],
+ vreinterpretq_s16_u16(vsubq_u16(v_src_08, v_pred_08)));
+ }
} else if (cols > 4) {
int r = rows;
do {
@@ -148,7 +306,27 @@ void aom_highbd_subtract_block_neon(int rows, int cols, int16_t *diff,
} while (r != 0);
} else {
int r = rows;
- do {
+ while (r >= 4) {
+ const uint16x8_t s01 =
+ vcombine_u16(vld1_u16(&src[0]), vld1_u16(&src[src_stride]));
+ const uint16x8_t p01 =
+ vcombine_u16(vld1_u16(&pred[0]), vld1_u16(&pred[pred_stride]));
+ const uint16x8_t s23 = vcombine_u16(vld1_u16(&src[2 * src_stride]),
+ vld1_u16(&src[3 * src_stride]));
+ const uint16x8_t p23 = vcombine_u16(vld1_u16(&pred[2 * pred_stride]),
+ vld1_u16(&pred[3 * pred_stride]));
+ const int16x8_t d01 = vreinterpretq_s16_u16(vsubq_u16(s01, p01));
+ const int16x8_t d23 = vreinterpretq_s16_u16(vsubq_u16(s23, p23));
+ vst1_s16(&diff[0], vget_low_s16(d01));
+ vst1_s16(&diff[diff_stride], vget_high_s16(d01));
+ vst1_s16(&diff[2 * diff_stride], vget_low_s16(d23));
+ vst1_s16(&diff[3 * diff_stride], vget_high_s16(d23));
+ diff += diff_stride << 2;
+ pred += pred_stride << 2;
+ src += src_stride << 2;
+ r -= 4;
+ }
+ while (r >= 2) {
const uint16x4_t v_src_r0 = vld1_u16(&src[0]);
const uint16x4_t v_src_r1 = vld1_u16(&src[src_stride]);
const uint16x4_t v_pred_r0 = vld1_u16(&pred[0]);
@@ -161,7 +339,7 @@ void aom_highbd_subtract_block_neon(int rows, int cols, int16_t *diff,
pred += pred_stride << 1;
src += src_stride << 1;
r -= 2;
- } while (r != 0);
+ }
}
}
#endif // CONFIG_AV1_HIGHBITDEPTH
diff --git a/aom_dsp/arm/sum_squares_neon.c b/aom_dsp/arm/sum_squares_neon.c
index 0c79fb6c20..eadab1aabb 100644
--- a/aom_dsp/arm/sum_squares_neon.c
+++ b/aom_dsp/arm/sum_squares_neon.c
@@ -24,17 +24,30 @@ static inline uint64_t aom_sum_squares_2d_i16_4x4_neon(const int16_t *src,
int16x4_t s2 = vld1_s16(src + 2 * stride);
int16x4_t s3 = vld1_s16(src + 3 * stride);
- int32x4_t sum_squares = vmull_s16(s0, s0);
- sum_squares = vmlal_s16(sum_squares, s1, s1);
- sum_squares = vmlal_s16(sum_squares, s2, s2);
- sum_squares = vmlal_s16(sum_squares, s3, s3);
+ int16x8_t s01 = vcombine_s16(s0, s1);
+ int16x8_t s23 = vcombine_s16(s2, s3);
+
+ int32x4_t sum_squares0 = vmull_s16(vget_low_s16(s01), vget_low_s16(s01));
+ int32x4_t sum_squares1 = vmull_s16(vget_low_s16(s23), vget_low_s16(s23));
+
+#if AOM_ARCH_AARCH64
+ sum_squares0 = vmlal_high_s16(sum_squares0, s01, s01);
+ sum_squares1 = vmlal_high_s16(sum_squares1, s23, s23);
+#else
+ sum_squares0 =
+ vmlal_s16(sum_squares0, vget_high_s16(s01), vget_high_s16(s01));
+ sum_squares1 =
+ vmlal_s16(sum_squares1, vget_high_s16(s23), vget_high_s16(s23));
+#endif
- return horizontal_long_add_u32x4(vreinterpretq_u32_s32(sum_squares));
+ return horizontal_long_add_u32x4(
+ vreinterpretq_u32_s32(vaddq_s32(sum_squares0, sum_squares1)));
}
static inline uint64_t aom_sum_squares_2d_i16_4xn_neon(const int16_t *src,
int stride, int height) {
- int32x4_t sum_squares[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };
+ int32x4_t sum_squares0 = vdupq_n_s32(0);
+ int32x4_t sum_squares1 = vdupq_n_s32(0);
int h = height;
do {
@@ -43,17 +56,29 @@ static inline uint64_t aom_sum_squares_2d_i16_4xn_neon(const int16_t *src,
int16x4_t s2 = vld1_s16(src + 2 * stride);
int16x4_t s3 = vld1_s16(src + 3 * stride);
- sum_squares[0] = vmlal_s16(sum_squares[0], s0, s0);
- sum_squares[0] = vmlal_s16(sum_squares[0], s1, s1);
- sum_squares[1] = vmlal_s16(sum_squares[1], s2, s2);
- sum_squares[1] = vmlal_s16(sum_squares[1], s3, s3);
+ int16x8_t s01 = vcombine_s16(s0, s1);
+ int16x8_t s23 = vcombine_s16(s2, s3);
+
+ sum_squares0 =
+ vmlal_s16(sum_squares0, vget_low_s16(s01), vget_low_s16(s01));
+ sum_squares1 =
+ vmlal_s16(sum_squares1, vget_low_s16(s23), vget_low_s16(s23));
+#if AOM_ARCH_AARCH64
+ sum_squares0 = vmlal_high_s16(sum_squares0, s01, s01);
+ sum_squares1 = vmlal_high_s16(sum_squares1, s23, s23);
+#else
+ sum_squares0 =
+ vmlal_s16(sum_squares0, vget_high_s16(s01), vget_high_s16(s01));
+ sum_squares1 =
+ vmlal_s16(sum_squares1, vget_high_s16(s23), vget_high_s16(s23));
+#endif
src += 4 * stride;
h -= 4;
} while (h != 0);
return horizontal_long_add_u32x4(
- vreinterpretq_u32_s32(vaddq_s32(sum_squares[0], sum_squares[1])));
+ vreinterpretq_u32_s32(vaddq_s32(sum_squares0, sum_squares1)));
}
static inline uint64_t aom_sum_squares_2d_i16_nxn_neon(const int16_t *src,
@@ -117,23 +142,34 @@ static inline uint64_t aom_sum_sse_2d_i16_4x4_neon(const int16_t *src,
int16x4_t s2 = vld1_s16(src + 2 * stride);
int16x4_t s3 = vld1_s16(src + 3 * stride);
- int32x4_t sse = vmull_s16(s0, s0);
- sse = vmlal_s16(sse, s1, s1);
- sse = vmlal_s16(sse, s2, s2);
- sse = vmlal_s16(sse, s3, s3);
+ int16x8_t s01 = vcombine_s16(s0, s1);
+ int16x8_t s23 = vcombine_s16(s2, s3);
+
+ int32x4_t sse_0 = vmull_s16(vget_low_s16(s01), vget_low_s16(s01));
+ int32x4_t sse_1 = vmull_s16(vget_low_s16(s23), vget_low_s16(s23));
- int32x4_t sum_01 = vaddl_s16(s0, s1);
- int32x4_t sum_23 = vaddl_s16(s2, s3);
- *sum += horizontal_add_s32x4(vaddq_s32(sum_01, sum_23));
+#if AOM_ARCH_AARCH64
+ sse_0 = vmlal_high_s16(sse_0, s01, s01);
+ sse_1 = vmlal_high_s16(sse_1, s23, s23);
+#else
+ sse_0 = vmlal_s16(sse_0, vget_high_s16(s01), vget_high_s16(s01));
+ sse_1 = vmlal_s16(sse_1, vget_high_s16(s23), vget_high_s16(s23));
+#endif
+
+ int32x4_t sum_acc = vpaddlq_s16(s01);
+ sum_acc = vpadalq_s16(sum_acc, s23);
+ *sum += horizontal_add_s32x4(sum_acc);
- return horizontal_long_add_u32x4(vreinterpretq_u32_s32(sse));
+ return horizontal_long_add_u32x4(
+ vreinterpretq_u32_s32(vaddq_s32(sse_0, sse_1)));
}
static inline uint64_t aom_sum_sse_2d_i16_4xn_neon(const int16_t *src,
int stride, int height,
int *sum) {
- int32x4_t sse[2] = { vdupq_n_s32(0), vdupq_n_s32(0) };
- int32x2_t sum_acc[2] = { vdup_n_s32(0), vdup_n_s32(0) };
+ int32x4_t sse_0 = vdupq_n_s32(0);
+ int32x4_t sse_1 = vdupq_n_s32(0);
+ int32x4_t sum_acc = vdupq_n_s32(0);
int h = height;
do {
@@ -142,23 +178,29 @@ static inline uint64_t aom_sum_sse_2d_i16_4xn_neon(const int16_t *src,
int16x4_t s2 = vld1_s16(src + 2 * stride);
int16x4_t s3 = vld1_s16(src + 3 * stride);
- sse[0] = vmlal_s16(sse[0], s0, s0);
- sse[0] = vmlal_s16(sse[0], s1, s1);
- sse[1] = vmlal_s16(sse[1], s2, s2);
- sse[1] = vmlal_s16(sse[1], s3, s3);
+ int16x8_t s01 = vcombine_s16(s0, s1);
+ int16x8_t s23 = vcombine_s16(s2, s3);
- sum_acc[0] = vpadal_s16(sum_acc[0], s0);
- sum_acc[0] = vpadal_s16(sum_acc[0], s1);
- sum_acc[1] = vpadal_s16(sum_acc[1], s2);
- sum_acc[1] = vpadal_s16(sum_acc[1], s3);
+ sse_0 = vmlal_s16(sse_0, vget_low_s16(s01), vget_low_s16(s01));
+ sse_1 = vmlal_s16(sse_1, vget_low_s16(s23), vget_low_s16(s23));
+#if AOM_ARCH_AARCH64
+ sse_0 = vmlal_high_s16(sse_0, s01, s01);
+ sse_1 = vmlal_high_s16(sse_1, s23, s23);
+#else
+ sse_0 = vmlal_s16(sse_0, vget_high_s16(s01), vget_high_s16(s01));
+ sse_1 = vmlal_s16(sse_1, vget_high_s16(s23), vget_high_s16(s23));
+#endif
+
+ sum_acc = vpadalq_s16(sum_acc, s01);
+ sum_acc = vpadalq_s16(sum_acc, s23);
src += 4 * stride;
h -= 4;
} while (h != 0);
- *sum += horizontal_add_s32x4(vcombine_s32(sum_acc[0], sum_acc[1]));
+ *sum += horizontal_add_s32x4(sum_acc);
return horizontal_long_add_u32x4(
- vreinterpretq_u32_s32(vaddq_s32(sse[0], sse[1])));
+ vreinterpretq_u32_s32(vaddq_s32(sse_0, sse_1)));
}
static inline uint64_t aom_sum_sse_2d_i16_nxn_neon(const int16_t *src,