Commit ba91c04786 for aom
commit ba91c0478630ba309dc6f5697f16532f32fc7f9e
Author: Frank Galligan <fgalligan@google.com>
Date: Sat Oct 31 09:51:49 2026 -0700
av1: Add AVX2 vectorization for quantize_b_qm
Vectorize two-stage exact quantization with quantization matrices
(qm_ptr != NULL) using AVX2 SIMD for both low-bit-depth and high-bit-
depth paths (av1_quantize_b_qm_avx2 and av1_highbd_quantize_b_qm_avx2).
Optimization details:
- Processes 8 coefficients per iteration using 256-bit AVX2 vectors.
- Shares parameter setup, DC-to-AC permutation, and EOB reduction
structure with av1_quantize_fp_qm_avx2.
- Preserves 100% bit-exact output parity with the scalar C helpers.
Benchmarking results (incremental vs prev CL / cumulative vs baseline):
- Stock avifenc still-image (Default 1T): -3.55% / -11.96% encode time
- Stock avifenc still-image (Default 8T): -3.19% / -10.22% encode time
- 48-image multi-rate corpus (Full Suite 1T): -4.12% / -13.05% time
- Bit-exact output (1T and 8T)
TAG=agy
CONV=b929a630-a6cd-45a0-979c-1767c30178ee
Change-Id: Ie225ac7cc59c713f40b7a38e698f69cc9ed91f1f
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl
index 46af4fb91d..6d3c3034e3 100644
--- a/av1/common/av1_rtcd_defs.pl
+++ b/av1/common/av1_rtcd_defs.pl
@@ -315,6 +315,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/void av1_quantize_fp_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
specialize qw/av1_quantize_fp_qm avx2/;
+ add_proto qw/void av1_quantize_b_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
+ specialize qw/av1_quantize_b_qm avx2/;
+
add_proto qw/void aom_quantize_b_helper/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, const int log_scale";
specialize qw/aom_quantize_b_helper neon/;
@@ -439,6 +442,9 @@ if (aom_config("CONFIG_AV1_ENCODER") eq "yes") {
add_proto qw/void av1_highbd_quantize_fp_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *round_ptr, const int16_t *quant_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
specialize qw/av1_highbd_quantize_fp_qm avx2/;
+
+ add_proto qw/void av1_highbd_quantize_b_qm/, "const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr, const int16_t *round_ptr, const int16_t *quant_ptr, const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, int log_scale";
+ specialize qw/av1_highbd_quantize_b_qm avx2/;
}
# End av1_high encoder functions
diff --git a/av1/encoder/av1_quantize.c b/av1/encoder/av1_quantize.c
index 9a3ccd5de0..8f9346a7cd 100644
--- a/av1/encoder/av1_quantize.c
+++ b/av1/encoder/av1_quantize.c
@@ -277,6 +277,19 @@ void av1_quantize_fp_qm_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
iscan, qm_ptr, iqm_ptr, log_scale);
}
+void av1_quantize_b_qm_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ const int16_t *zbin_ptr, const int16_t *round_ptr,
+ const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr,
+ uint16_t *eob_ptr, const int16_t *scan,
+ const int16_t *iscan, const qm_val_t *qm_ptr,
+ const qm_val_t *iqm_ptr, int log_scale) {
+ aom_quantize_b_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr,
+ quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr,
+ eob_ptr, scan, iscan, qm_ptr, iqm_ptr, log_scale);
+}
+
void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
@@ -355,10 +368,10 @@ void av1_quantize_b_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
#endif // !CONFIG_REALTIME_ONLY
if (qm_ptr != NULL && iqm_ptr != NULL) {
- aom_quantize_b_helper_c(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX,
- p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr,
- dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
- sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
+ av1_quantize_b_qm(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX,
+ p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
+ p->dequant_QTX, eob_ptr, sc->scan, sc->iscan, qm_ptr,
+ iqm_ptr, qparam->log_scale);
} else {
switch (qparam->log_scale) {
case 0:
@@ -497,10 +510,10 @@ void av1_highbd_quantize_b_facade(const tran_low_t *coeff_ptr,
#endif // !CONFIG_REALTIME_ONLY
if (qm_ptr != NULL && iqm_ptr != NULL) {
- aom_highbd_quantize_b_helper_c(
- coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
- p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
- sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
+ av1_highbd_quantize_b_qm(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX,
+ p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr,
+ dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
+ sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
} else {
switch (qparam->log_scale) {
case 0:
@@ -599,6 +612,19 @@ void av1_highbd_quantize_fp_qm_c(
NULL, qcoeff_ptr, dqcoeff_ptr, dequant_ptr,
eob_ptr, scan, iscan, qm_ptr, iqm_ptr, log_scale);
}
+
+void av1_highbd_quantize_b_qm_c(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
+ const int16_t *round_ptr, const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
+ const qm_val_t *iqm_ptr, int log_scale) {
+ aom_highbd_quantize_b_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr,
+ quant_ptr, quant_shift_ptr, qcoeff_ptr,
+ dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan,
+ qm_ptr, iqm_ptr, log_scale);
+}
#endif // CONFIG_AV1_HIGHBITDEPTH
static void invert_quant(int16_t *quant, int16_t *shift, int d) {
diff --git a/av1/encoder/x86/av1_quantize_avx2.c b/av1/encoder/x86/av1_quantize_avx2.c
index fb2eeb9807..29e3868f38 100644
--- a/av1/encoder/x86/av1_quantize_avx2.c
+++ b/av1/encoder/x86/av1_quantize_avx2.c
@@ -589,3 +589,196 @@ void av1_highbd_quantize_fp_qm_avx2(
iscan_ptr, qm_ptr, iqm_ptr, log_scale, 1);
}
#endif
+
+static inline void init_qp_b_qm(const int16_t *zbin_ptr,
+ const int16_t *round_ptr,
+ const int16_t *quant_ptr,
+ const int16_t *dequant_ptr,
+ const int16_t *quant_shift_ptr, int log_scale,
+ __m256i *zbin, __m256i *qp) {
+ const __m128i zbin_128 = _mm_loadu_si128((const __m128i *)zbin_ptr);
+ const __m128i round = _mm_loadu_si128((const __m128i *)round_ptr);
+ const __m128i quant = _mm_loadu_si128((const __m128i *)quant_ptr);
+ const __m128i dequant = _mm_loadu_si128((const __m128i *)dequant_ptr);
+ const __m128i quant_shift = _mm_loadu_si128((const __m128i *)quant_shift_ptr);
+
+ // Sign-extend 16-bit quantizer parameters (lane 0: DC, lanes 1..7: AC) to
+ // 32-bit lanes.
+ *zbin = _mm256_cvtepi16_epi32(zbin_128);
+ qp[0] = _mm256_cvtepi16_epi32(round);
+ qp[1] = _mm256_cvtepi16_epi32(quant);
+ qp[2] = _mm256_cvtepi16_epi32(dequant);
+ qp[3] = _mm256_cvtepi16_epi32(quant_shift);
+
+ if (log_scale > 0) {
+ const __m256i rnd = _mm256_set1_epi32(1 << (log_scale - 1));
+ *zbin = _mm256_add_epi32(*zbin, rnd);
+ *zbin = _mm256_srai_epi32(*zbin, log_scale);
+
+ qp[0] = _mm256_add_epi32(qp[0], rnd);
+ qp[0] = _mm256_srai_epi32(qp[0], log_scale);
+ }
+
+ *zbin = _mm256_slli_epi32(*zbin, AOM_QM_BITS);
+ // Subtracting 1 here eliminates a _mm256_cmpeq_epi32() instruction when
+ // calculating the zbin mask.
+ *zbin = _mm256_sub_epi32(*zbin, _mm256_set1_epi32(1));
+}
+
+static inline void update_qp_b_qm(__m256i *zbin, __m256i *qp) {
+ update_qp(zbin, qp);
+ qp[3] = _mm256_permute2x128_si256(qp[3], qp[3], 0x11);
+}
+
+static AOM_FORCE_INLINE void quantize_b_qm_8(
+ const __m256i *zbin, const __m256i *qp, const tran_low_t *coeff_ptr,
+ const int16_t *iscan_ptr, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr, __m128i shift,
+ __m128i log_scale_vec, int is_highbd, __m256i *eob) {
+ // Load 8 coefficients and 8 quantization matrix weights, then check if any
+ // weighted coefficient exceeds the zero-bin threshold.
+ const __m256i coeff = _mm256_loadu_si256((const __m256i *)coeff_ptr);
+ const __m256i abs_coeff = _mm256_abs_epi32(coeff);
+ const __m256i wt =
+ _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)qm_ptr));
+ const __m256i abs_wt =
+ is_highbd
+ ? _mm256_mullo_epi32(abs_coeff, wt)
+ : _mm256_madd_epi16(
+ _mm256_min_epi32(abs_coeff, _mm256_set1_epi32(INT16_MAX)), wt);
+ const __m256i mask = _mm256_cmpgt_epi32(abs_wt, *zbin);
+ const int nzflag = _mm256_movemask_epi8(mask);
+
+ if (nzflag) {
+ // Round and apply forward quantization matrix weights. For low bit-depth,
+ // clamp the rounded coefficient to INT16_MAX to match the C reference.
+ __m256i tmp_rnd = _mm256_add_epi32(abs_coeff, qp[0]);
+ if (!is_highbd) {
+ tmp_rnd = _mm256_min_epi32(tmp_rnd, _mm256_set1_epi32(INT16_MAX));
+ }
+ const __m256i tmp_wt = is_highbd ? _mm256_mullo_epi32(tmp_rnd, wt)
+ : _mm256_madd_epi16(tmp_rnd, wt);
+
+ // First stage of exact division: tmp2 = ((tmp_wt * quant) >> 16) + tmp_wt.
+ const __m256i q_lo = _mm256_mul_epi32(tmp_wt, qp[1]);
+ const __m256i sh_lo = _mm256_srli_epi64(q_lo, 16);
+ const __m256i q_hi = _mm256_mul_epi32(_mm256_srli_epi64(tmp_wt, 32),
+ _mm256_srli_epi64(qp[1], 32));
+ const __m256i sh_hi = _mm256_slli_epi64(q_hi, 16);
+ const __m256i tmp2 =
+ _mm256_add_epi32(_mm256_blend_epi32(sh_lo, sh_hi, 0xAA), tmp_wt);
+
+ // Second stage of exact division:
+ // abs_q = (tmp2 * quant_shift) >> (16 - log_scale + AOM_QM_BITS).
+ const __m256i s_lo = _mm256_mul_epu32(tmp2, qp[3]);
+ const __m256i r_lo = _mm256_srl_epi64(s_lo, shift);
+ const __m256i s_hi = _mm256_mul_epu32(_mm256_srli_epi64(tmp2, 32),
+ _mm256_srli_epi64(qp[3], 32));
+ const __m256i r_hi = _mm256_slli_epi64(_mm256_srl_epi64(s_hi, shift), 32);
+ const __m256i abs_q = _mm256_and_si256(_mm256_or_si256(r_lo, r_hi), mask);
+
+ // Dequantize using the inverse quantization matrix weights (iqm_ptr).
+ const __m256i iwt =
+ _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i *)iqm_ptr));
+ const __m256i dequant = _mm256_srai_epi32(
+ _mm256_add_epi32(_mm256_madd_epi16(qp[2], iwt),
+ _mm256_set1_epi32(1 << (AOM_QM_BITS - 1))),
+ AOM_QM_BITS);
+ const __m256i abs_dq =
+ _mm256_sra_epi32(_mm256_mullo_epi32(abs_q, dequant), log_scale_vec);
+
+ // Restore original signs and store quantized and dequantized coefficients.
+ const __m256i q = _mm256_sign_epi32(abs_q, coeff);
+ const __m256i dq = _mm256_sign_epi32(abs_dq, coeff);
+ const __m256i nz_mask = _mm256_cmpgt_epi32(abs_q, _mm256_setzero_si256());
+
+ _mm256_storeu_si256((__m256i *)qcoeff_ptr, q);
+ _mm256_storeu_si256((__m256i *)dqcoeff_ptr, dq);
+
+ *eob = get_max_lane_eob_qm(iscan_ptr, *eob, nz_mask);
+ } else {
+ const __m256i zero = _mm256_setzero_si256();
+ _mm256_storeu_si256((__m256i *)qcoeff_ptr, zero);
+ _mm256_storeu_si256((__m256i *)dqcoeff_ptr, zero);
+ }
+}
+
+static AOM_FORCE_INLINE void quantize_b_qm_helper_avx2(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
+ const int16_t *round_ptr, const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *iscan_ptr, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale, int is_highbd) {
+ const int step = 8;
+ const __m128i shift = _mm_cvtsi32_si128(16 - log_scale + AOM_QM_BITS);
+ const __m128i log_scale_vec = _mm_cvtsi32_si128(log_scale);
+ __m256i qp[4], zbin;
+ __m256i eob = _mm256_setzero_si256();
+
+ init_qp_b_qm(zbin_ptr, round_ptr, quant_ptr, dequant_ptr, quant_shift_ptr,
+ log_scale, &zbin, qp);
+
+ // Process DC and the first 7 AC coeffs.
+ quantize_b_qm_8(&zbin, qp, coeff_ptr, iscan_ptr, qm_ptr, iqm_ptr, qcoeff_ptr,
+ dqcoeff_ptr, shift, log_scale_vec, is_highbd, &eob);
+
+ coeff_ptr += step;
+ qcoeff_ptr += step;
+ dqcoeff_ptr += step;
+ iscan_ptr += step;
+ qm_ptr += step;
+ iqm_ptr += step;
+ n_coeffs -= step;
+
+ // Overwrite the DC constants with AC constants.
+ update_qp_b_qm(&zbin, qp);
+
+ // AC only loop.
+ while (n_coeffs > 0) {
+ quantize_b_qm_8(&zbin, qp, coeff_ptr, iscan_ptr, qm_ptr, iqm_ptr,
+ qcoeff_ptr, dqcoeff_ptr, shift, log_scale_vec, is_highbd,
+ &eob);
+
+ coeff_ptr += step;
+ qcoeff_ptr += step;
+ dqcoeff_ptr += step;
+ iscan_ptr += step;
+ qm_ptr += step;
+ iqm_ptr += step;
+ n_coeffs -= step;
+ }
+ *eob_ptr = quant_gather_eob(eob);
+}
+
+void av1_quantize_b_qm_avx2(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
+ const int16_t *zbin_ptr, const int16_t *round_ptr,
+ const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr,
+ tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
+ const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan_ptr, const int16_t *iscan_ptr,
+ const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
+ int log_scale) {
+ (void)scan_ptr;
+ quantize_b_qm_helper_avx2(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr,
+ quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr,
+ dequant_ptr, eob_ptr, iscan_ptr, qm_ptr, iqm_ptr,
+ log_scale, 0);
+}
+
+#if CONFIG_AV1_HIGHBITDEPTH
+void av1_highbd_quantize_b_qm_avx2(
+ const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
+ const int16_t *round_ptr, const int16_t *quant_ptr,
+ const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
+ tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
+ const int16_t *scan_ptr, const int16_t *iscan_ptr, const qm_val_t *qm_ptr,
+ const qm_val_t *iqm_ptr, int log_scale) {
+ (void)scan_ptr;
+ quantize_b_qm_helper_avx2(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr,
+ quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr,
+ dequant_ptr, eob_ptr, iscan_ptr, qm_ptr, iqm_ptr,
+ log_scale, 1);
+}
+#endif
diff --git a/test/quantize_func_test.cc b/test/quantize_func_test.cc
index 0c7cc123b7..8c89af2293 100644
--- a/test/quantize_func_test.cc
+++ b/test/quantize_func_test.cc
@@ -485,6 +485,8 @@ using QuantizeFpQmFunc = void (*)(
const int16_t *dequant_ptr, uint16_t *eob_ptr, const int16_t *scan,
const int16_t *iscan, const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr,
int log_scale);
+using QuantizeBQmFunc = void (*)(QUAN_PARAM_LIST, const qm_val_t *qm_ptr,
+ const qm_val_t *iqm_ptr, int log_scale);
const CommonQuantParams *GetQmParams() {
static CommonQuantParams qm_params;
@@ -507,6 +509,17 @@ void quan_fp_qm_wrapper(QUAN_PARAM_LIST) {
fn(coeff_ptr, n_coeffs, round_ptr, quant_ptr, qcoeff_ptr, dqcoeff_ptr,
dequant_ptr, eob_ptr, scan, iscan, qm_ptr, iqm_ptr, log_scale);
}
+
+template <QuantizeBQmFunc fn, TX_SIZE tx_size, int qm_level = 5>
+void quan_b_qm_wrapper(QUAN_PARAM_LIST) {
+ const CommonQuantParams *qm_params = GetQmParams();
+ const qm_val_t *qm_ptr = qm_params->gqmatrix[qm_level][0][tx_size];
+ const qm_val_t *iqm_ptr = qm_params->giqmatrix[qm_level][0][tx_size];
+ const int log_scale = av1_get_tx_scale(tx_size);
+ fn(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr, quant_shift_ptr,
+ qcoeff_ptr, dqcoeff_ptr, dequant_ptr, eob_ptr, scan, iscan, qm_ptr,
+ iqm_ptr, log_scale);
+}
#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
const QuantizeParam<LPQuantizeFunc> kLPQParamArrayAvx2[] = {
@@ -618,6 +631,35 @@ const QuantizeParam<QuantizeFunc> kQParamArrayAvx2[] = {
static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_12),
make_tuple(&aom_highbd_quantize_b_64x64_c, &aom_highbd_quantize_b_64x64_avx2,
static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_12),
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_10),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_16X16>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_12),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_10),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_32X32>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_12),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_10),
+ make_tuple(&quan_b_qm_wrapper<av1_highbd_quantize_b_qm_c, TX_64X64>,
+ &quan_b_qm_wrapper<av1_highbd_quantize_b_qm_avx2, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_12),
+#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
#if !CONFIG_REALTIME_ONLY
make_tuple(&aom_highbd_quantize_b_adaptive_c,
&aom_highbd_quantize_b_adaptive_avx2,
@@ -653,6 +695,17 @@ const QuantizeParam<QuantizeFunc> kQParamArrayAvx2[] = {
static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
make_tuple(&aom_quantize_b_64x64_c, &aom_quantize_b_64x64_avx2,
static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+#if CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
+ make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_16X16>,
+ &quan_b_qm_wrapper<av1_quantize_b_qm_avx2, TX_16X16>,
+ static_cast<TX_SIZE>(TX_16X16), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_32X32>,
+ &quan_b_qm_wrapper<av1_quantize_b_qm_avx2, TX_32X32>,
+ static_cast<TX_SIZE>(TX_32X32), TYPE_B, AOM_BITS_8),
+ make_tuple(&quan_b_qm_wrapper<av1_quantize_b_qm_c, TX_64X64>,
+ &quan_b_qm_wrapper<av1_quantize_b_qm_avx2, TX_64X64>,
+ static_cast<TX_SIZE>(TX_64X64), TYPE_B, AOM_BITS_8),
+#endif // CONFIG_QUANT_MATRIX || CONFIG_AV1_DECODER
};
INSTANTIATE_TEST_SUITE_P(AVX2, FullPrecisionQuantizeTest,