Commit df49183d1a for aom
commit df49183d1a51d67eb2e6c12e3a256dd4a04e878b
Author: Lin Zheng <linzhen@google.com>
Date: Thu Sep 17 21:28:29 2026 +0000
Adjust transform search and skip decisions for HBD sharpness
Disable pre-skip prediction when reconstructed variance drops below
source variance, and bias against skip txfm when skipping the transform
causes texutre loss.
Bug: 548701298
Change-Id: Idd0298cf4f912880ec68c79f02c840677ae98dde
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index b0ea57449a..d6b6e2057b 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -644,8 +644,8 @@ static void get_variance_stats_hbd(const MACROBLOCK *x, int64_t *src_var,
}
#endif // CONFIG_AV1_HIGHBITDEPTH
-static void get_variance_stats(const MACROBLOCK *x, int64_t *src_var,
- int64_t *rec_var) {
+void av1_get_variance_stats(const MACROBLOCK *x, int64_t *src_var,
+ int64_t *rec_var) {
#if CONFIG_AV1_HIGHBITDEPTH
if (is_cur_buf_hbd(&x->e_mbd)) {
get_variance_stats_hbd(x, src_var, rec_var);
@@ -665,6 +665,54 @@ static void get_variance_stats(const MACROBLOCK *x, int64_t *src_var,
*src_var = aom_calc_variance_stat(p->src.buf, p->src.stride, bw, bh);
}
+int av1_is_skip_txfm_penalized(const AV1_COMP *cpi, const MACROBLOCK *x,
+ BLOCK_SIZE bsize) {
+#if CONFIG_AV1_HIGHBITDEPTH
+ const MACROBLOCKD *xd = &x->e_mbd;
+ if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3 &&
+ !frame_is_kf_gf_arf(cpi)) {
+ int64_t src_var, rec_var;
+ av1_get_variance_stats(x, &src_var, &rec_var);
+ if (src_var > rec_var) {
+ const int num_pixels = block_size_wide[bsize] * block_size_high[bsize];
+ const int64_t src_var_per_px = src_var / num_pixels;
+ if (src_var_per_px >= 0) return 1;
+ }
+ }
+#else
+ (void)cpi;
+ (void)x;
+ (void)bsize;
+#endif
+ return 0;
+}
+
+void av1_get_tx_skip_dist(const AV1_COMP *cpi, const MACROBLOCK *x,
+ BLOCK_SIZE bsize, int64_t dist, int64_t sse,
+ int64_t *no_skip_dist, int64_t *skip_dist) {
+ *no_skip_dist = dist;
+ *skip_dist = sse;
+#if CONFIG_AV1_HIGHBITDEPTH
+ const MACROBLOCKD *xd = &x->e_mbd;
+ if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3 &&
+ !frame_is_kf_gf_arf(cpi)) {
+ int64_t src_var, rec_var;
+ av1_get_variance_stats(x, &src_var, &rec_var);
+ if (src_var > rec_var) {
+ int64_t var_offset = src_var - rec_var;
+ const int num_pixels = block_size_wide[bsize] * block_size_high[bsize];
+ const int64_t src_var_per_px = src_var / num_pixels;
+ *no_skip_dist += var_offset;
+ *skip_dist += (src_var_per_px >= 0) ? (var_offset * 4) : var_offset;
+ }
+ }
+#else
+ (void)cpi;
+ (void)x;
+ (void)bsize;
+#endif
+}
+
static void adjust_rdcost(const AV1_COMP *cpi, const MACROBLOCK *x,
RD_STATS *rd_cost, bool is_inter_pred) {
if ((cpi->oxcf.tune_cfg.tuning == AOM_TUNE_IQ ||
@@ -693,7 +741,7 @@ static void adjust_rdcost(const AV1_COMP *cpi, const MACROBLOCK *x,
if (frame_is_kf_gf_arf(cpi)) return;
int64_t src_var, rec_var;
- get_variance_stats(x, &src_var, &rec_var);
+ av1_get_variance_stats(x, &src_var, &rec_var);
if (src_var <= rec_var) return;
@@ -718,7 +766,7 @@ static void adjust_cost(const AV1_COMP *cpi, const MACROBLOCK *x,
if (frame_is_kf_gf_arf(cpi)) return;
int64_t src_var, rec_var;
- get_variance_stats(x, &src_var, &rec_var);
+ av1_get_variance_stats(x, &src_var, &rec_var);
if (src_var <= rec_var) return;
diff --git a/av1/encoder/rdopt_utils.h b/av1/encoder/rdopt_utils.h
index 04d46d01b5..208216327d 100644
--- a/av1/encoder/rdopt_utils.h
+++ b/av1/encoder/rdopt_utils.h
@@ -901,6 +901,16 @@ static inline int get_txfm_rd_gate_level(
return txfm_rd_gate_level[TX_SEARCH_DEFAULT];
}
+void av1_get_variance_stats(const MACROBLOCK *x, int64_t *src_var,
+ int64_t *rec_var);
+
+void av1_get_tx_skip_dist(const struct AV1_COMP *cpi, const MACROBLOCK *x,
+ BLOCK_SIZE bsize, int64_t dist, int64_t sse,
+ int64_t *no_skip_dist, int64_t *skip_dist);
+
+int av1_is_skip_txfm_penalized(const struct AV1_COMP *cpi, const MACROBLOCK *x,
+ BLOCK_SIZE bsize);
+
#ifdef __cplusplus
} // extern "C"
#endif
diff --git a/av1/encoder/tx_search.c b/av1/encoder/tx_search.c
index 70e862c743..6281191572 100644
--- a/av1/encoder/tx_search.c
+++ b/av1/encoder/tx_search.c
@@ -190,12 +190,20 @@ static inline int64_t pixel_diff_stats(
// Uses simple features on top of DCT coefficients to quickly predict
// whether optimal RD decision is to skip encoding the residual.
// The sse value is stored in dist.
-static int predict_skip_txfm(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist,
+static int predict_skip_txfm(const AV1_COMP *cpi, MACROBLOCK *x,
+ BLOCK_SIZE bsize, int64_t *dist,
int reduced_tx_set) {
+ const MACROBLOCKD *xd = &x->e_mbd;
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) {
+ if (av1_is_skip_txfm_penalized(cpi, x, bsize)) return 0;
+ }
+#else
+ (void)cpi;
+#endif
const TxfmSearchParams *txfm_params = &x->txfm_search_params;
const int bw = block_size_wide[bsize];
const int bh = block_size_high[bsize];
- const MACROBLOCKD *xd = &x->e_mbd;
const int16_t dc_q = av1_dc_quant_QTX(x->qindex, 0, xd->bd);
*dist = av1_pixel_diff_dist(x, 0, 0, 0, bsize, bsize, NULL);
@@ -2953,13 +2961,33 @@ static int64_t uniform_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
}
// Check if forcing the block to skip transform leads to smaller RD cost.
if (is_inter && !rd_stats->skip_txfm && !xd->lossless[mbmi->segment_id]) {
- int64_t temp_skip_txfm_rd =
- RDCOST(x->rdmult, skip_txfm_rate, rd_stats->sse);
- if (temp_skip_txfm_rd <= rd) {
- rd = temp_skip_txfm_rd;
- rd_stats->rate = 0;
- rd_stats->dist = rd_stats->sse;
- rd_stats->skip_txfm = 1;
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) {
+ int64_t no_skip_dist, skip_dist;
+ av1_get_tx_skip_dist(cpi, x, bs, rd_stats->dist, rd_stats->sse,
+ &no_skip_dist, &skip_dist);
+ const int64_t temp_skip_txfm_rd =
+ RDCOST(x->rdmult, skip_txfm_rate, skip_dist);
+ const int64_t temp_no_skip_txfm_rd =
+ RDCOST(x->rdmult, rd_stats->rate + no_skip_txfm_rate + tx_size_rate,
+ no_skip_dist);
+ if (temp_skip_txfm_rd <= temp_no_skip_txfm_rd) {
+ rd = temp_skip_txfm_rd;
+ rd_stats->rate = 0;
+ rd_stats->dist = rd_stats->sse;
+ rd_stats->skip_txfm = 1;
+ }
+ } else
+#endif
+ {
+ const int64_t temp_skip_txfm_rd =
+ RDCOST(x->rdmult, skip_txfm_rate, rd_stats->sse);
+ if (temp_skip_txfm_rd <= rd) {
+ rd = temp_skip_txfm_rd;
+ rd_stats->rate = 0;
+ rd_stats->dist = rd_stats->sse;
+ rd_stats->skip_txfm = 1;
+ }
}
}
@@ -3600,7 +3628,7 @@ void av1_pick_recursive_tx_size_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
// context and terminate early.
int64_t dist;
if (txfm_params->skip_txfm_level &&
- predict_skip_txfm(x, bsize, &dist,
+ predict_skip_txfm(cpi, x, bsize, &dist,
cpi->common.features.reduced_tx_set_used)) {
set_skip_txfm(x, rd_stats, bsize, dist);
// Save the RD search results into mb_rd_record.
@@ -3673,7 +3701,7 @@ void av1_pick_uniform_tx_size_type_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
int64_t dist;
if (tx_params->skip_txfm_level && is_inter &&
!xd->lossless[mbmi->segment_id] &&
- predict_skip_txfm(x, bs, &dist,
+ predict_skip_txfm(cpi, x, bs, &dist,
cpi->common.features.reduced_tx_set_used)) {
// Populate rdstats as per skip decision
set_skip_txfm(x, rd_stats, bs, dist);
@@ -3874,14 +3902,23 @@ int av1_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
int choose_skip_txfm = rd_stats->skip_txfm;
if (!choose_skip_txfm && !xd->lossless[mbmi->segment_id]) {
+ int64_t no_skip_dist = rd_stats->dist;
+ int64_t skip_dist = rd_stats->sse;
+#if CONFIG_AV1_HIGHBITDEPTH
+ if (xd->bd > 8 && cpi->oxcf.algo_cfg.sharpness == 3) {
+ av1_get_tx_skip_dist(cpi, x, bsize, rd_stats->dist, rd_stats->sse,
+ &no_skip_dist, &skip_dist);
+ }
+#endif
const int64_t rdcost_no_skip_txfm = RDCOST(
x->rdmult, rd_stats_y->rate + rd_stats_uv->rate + skip_txfm_cost[0],
- rd_stats->dist);
+ no_skip_dist);
const int64_t rdcost_skip_txfm =
- RDCOST(x->rdmult, skip_txfm_cost[1], rd_stats->sse);
+ RDCOST(x->rdmult, skip_txfm_cost[1], skip_dist);
if (rdcost_no_skip_txfm >= rdcost_skip_txfm) choose_skip_txfm = 1;
}
if (choose_skip_txfm) {
+ const int naturally_skip = rd_stats->skip_txfm;
rd_stats_y->rate = 0;
rd_stats_uv->rate = 0;
rd_stats->rate = mode_rate + skip_txfm_cost[1];
@@ -3889,13 +3926,15 @@ int av1_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
rd_stats_y->dist = rd_stats_y->sse;
rd_stats_uv->dist = rd_stats_uv->sse;
mbmi->skip_txfm = 1;
- if (rd_stats->skip_txfm) {
+ rd_stats->skip_txfm = 1;
+ if (naturally_skip) {
const int64_t tmprd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist);
if (tmprd > ref_best_rd) return 0;
}
} else {
rd_stats->rate += skip_txfm_cost[0];
mbmi->skip_txfm = 0;
+ rd_stats->skip_txfm = 0;
}
return 1;