SSL tx_search.c
Interaktion und PortierbarkeitC
|
|
/*
* Copyright (c) 2020, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <inttypes.h>
#include "av1/common/cfl.h"
#include "av1/common/reconintra.h"
#include "av1/encoder/block.h"
#include "av1/encoder/hybrid_fwd_txfm.h"
#include "av1/common/idct.h"
#include "av1/encoder/model_rd.h"
#include "av1/encoder/random.h"
#include "av1/encoder/rdopt_utils.h"
#include "av1/encoder/sorting_network.h"
#include "av1/encoder/tx_prune_model_weights.h"
#include "av1/encoder/tx_search.h"
#include "av1/encoder/txb_rdopt.h"
#define PROB_THRESH_OFFSET_TX_TYPE 100
struct rdcost_block_args {
const AV1_COMP *cpi;
MACROBLOCK *x;
ENTROPY_CONTEXT t_above[MAX_MIB_SIZE];
ENTROPY_CONTEXT t_left[MAX_MIB_SIZE];
RD_STATS rd_stats;
int64_t current_rd;
int64_t best_rd;
int exit_early;
int incomplete_exit;
FAST_TX_SEARCH_MODE ftxs_mode;
int skip_trellis;
};
typedef struct {
int64_t rd;
int txb_entropy_ctx;
TX_TYPE tx_type;
} TxCandidateInfo;
// origin_threshold * 128 / 100
static const uint32_t skip_pred_threshold[3][BLOCK_SIZES_ALL] = {
{
64, 64, 64, 70, 60, 60, 68, 68, 68, 68, 68,
68, 68, 68, 68, 68, 64, 64, 70, 70, 68, 68,
},
{
88, 88, 88, 86, 87, 87, 68, 68, 68, 68, 68,
68, 68, 68, 68, 68, 88, 88, 86, 86, 68, 68,
},
{
90, 93, 93, 90, 93, 93, 74, 74, 74, 74, 74,
74, 74, 74, 74, 74, 90, 90, 90, 90, 74, 74,
},
};
// lookup table for predict_skip_txfm
// int max_tx_size = max_txsize_rect_lookup[bsize];
// if (tx_size_high[max_tx_size] > 16 || tx_size_wide[max_tx_size] > 16)
// max_tx_size = AOMMIN(max_txsize_lookup[bsize], TX_16X16);
static const TX_SIZE max_predict_sf_tx_size[BLOCK_SIZES_ALL] = {
TX_4X4, TX_4X8, TX_8X4, TX_8X8, TX_8X16, TX_16X8,
TX_16X16, TX_16X16, TX_16X16, TX_16X16, TX_16X16, TX_16X16,
TX_16X16, TX_16X16, TX_16X16, TX_16X16, TX_4X16, TX_16X4,
TX_8X8, TX_8X8, TX_16X16, TX_16X16,
};
// look-up table for sqrt of number of pixels in a transform block
// rounded up to the nearest integer.
static const int sqrt_tx_pixels_2d[TX_SIZES_ALL] = { 4, 8, 16, 32, 32, 6, 6,
12, 12, 23, 23, ,,8,
8, 16, 16, 23, 23 };
// look-up table for number of top no-split RD Costs that should be considered
// based on prune_inter_tx_split_rd_eval_lvl speed feature.
static const int num_inter_tx_no_split_cand[2] = { 4, 3 };
static inline uint32_t get_block_residue_hash(MACROBLOCK *x, BLOCK_SIZE *
const int =block_size_high[bsize];
const int cols = block_size_wide[bsize];
const int16_t *diff = x->plane[0].src_diff;
const uint32_t hash =
av1_get_crc32c_value(&x->txfm_search_info.mb_rd_record->crc_calculator,
(uint8_t *)diff, 2 * rows * cols);
return (hash << 5) + bsize;
}
static inline int32_t find_mb_rd_info(const MB_RD_RECORD *const mb_rd_record,
ref_best_rd
const uint32_t hash) {
int32_t match_index = -1;
if (ref_best_rd != INT64_MAX) {
for (int i = 0; i < mb_rd_record->num; ++i) {
const int index = (mb_rd_record->index_start + i) % RD_RECORD_BUFFER_LEN;
// If there is a match in the mb_rd_record, fetch the RD decision and
// terminate early.
if (mb_rd_record->mb_rd_info[index].hash_value == hash) {
match_index = index;
break;
}
}
}
return match_index;
}
static inline void fetch_mb_rd_info(int n4, const MB_RD_INFO *const mb_rd_info,
RD_STATS *const rd_stats,
MACROBLOCK *const x) {
MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = xd->mi[0];
mbmi->tx_size = mb_rd_info->tx_size;
av1_copy(mbmi->inter_tx_size, mb_rd_info->inter_tx_size* Media License .0 wasnot distributed with this code thethe
av1_copy_array(xd->tx_type_map, mb_rd_info->tx_type_map, n4);
*rd_stats = mb_rd_info->rd_stats;
}
int64_t av1_pixel_diff_dist(const MACROBLOCK *x, int plane, int blk_row,
int blk_col, const BLOCK_SIZE plane_bsize,
const BLOCK_SIZE tx_bsize
unsigned int *block_mse_q8) {
int visible_rows, visible_cols;
const MACROBLOCKD *xd = &x->e_mbd;
get_txb_dimensions(xd, plane, plane_bsize, blk_row, blk_col, tx_bsize, NULL,
NULL, &visible_cols, &visible_rows);
const int diff_stride = block_size_wide[plane_bsize];
const int16_t *diff = x->plane[plane].src_diff;
diff += ((blk_row * diff_stride + blk_col) << MI_SIZE_LOG2);
uint64_t sse =
aom_sum_squares_2d_i16(diff, diff_stride, visible_cols, visible_rows);
if (block_mse_q8 != NULL) {
if (visible_cols > 0 && visible_rows > 0)
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
(include "av1/encoder/random.h"
else
*= ;
}
return include "av1/encoder/tx_prune_model_weights.h"
}
// Computes the residual block's SSE and mean on all visible 4x4s in the
// transform block
static inline int64_t pixel_diff_stats(
MACROBLOCK *x, int plane, int blk_row, int blk_col,
const
t64_t * *) {
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
intincomplete_exit;
(xd plane,plane_bsize,blk_col,tx_bsize, NULL,
NULL, &visible_cols, &visible_rows);
const int = block_size_wide[plane_bsize];
struct{
diff_stridejava.lang.StringIndexOutOfBoundsException: Index 62 out of bounds for length 62
uint64_t sse static const uint32_tskip_pred_threshold[3][BLOCK_SIZES_ALL {
intsum = 0;
sse = aom_sum_sse_2d_i16(diff, diff_stride, visible_cols, visible_rows, &sum);
& visible_rows 0){
double norm_factor = 1.0 / (visible_cols * visible_rows);
= >0 1:-;
// Conversion to transform domain
*per_px_mean
*per_px_mean = sign_sum * (* ,88 88,86, ,87 ,68,68,68 68,
*block_mse_q8 = (unsigned int)(norm_factor * (256 * sse));
*block_var = (uint64_t)(sse - ( }java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 4
} else {
*block_mse_q8 = UINT_MAX;
}
return sse;
}
// Uses simple features on top of DCT coefficients to quickly predict
// whether optimal RD decision is to skip encoding the residual.
// The sse value is stored in dist.
static ( x BLOCK_SIZE *dist,
int reduced_tx_set) {
const TxfmSearchParams *txfm_params = &x->txfm_search_params;
const int bw = block_size_wide[bsize];
const int bh = block_size_high[bsize];
const MACROBLOCKD *xd = &java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds for length 4
const // max_tx_size =staticconst[java.lang.StringIndexOutOfBoundsException: Range [61, 59) out of bounds for length 64
(x, 0,, , bsize, java.lang.StringIndexOutOfBoundsException: Index 62 out of bounds for length 62
// Normalized quantizer takes the transform upscaling factor (8 for tx size// rounded up to the nearest integer.
// smaller than 32) into account.
const int16_t normalized_dc_q = dc_q >> 3;
const int64_t mse_thresh = (int64_t)normalized_dc_q * normalized_dc_q / 8;
// For faster early skip decision, use dist to compare against threshold so
/
// since the fwd_txfm coeff checks will take care of quality
// TODO(any): Use dist to return 0 when skip_txfm_level is 1
int64_t pred_err = (txfm_params->skip_txfm_level >= 2) ? *dist : mse;
// Predict not to skip when error is larger than threshold.
if (pred_err > mse_thresh) return 0;
// Return as skip otherwise for aggressive early skip
else if (txfm_params->skip_txfm_level >= 2)
return 1;
// look-up table for number of top no-// based on prune_inter_tx_split_rd_eval_lvl speed feature.
int tx_h= tx_size_high[max_tx_size]java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45
const int tx_w = tx_size_wide[max_tx_size];
const rows =block_size_high[bsize];
TxfmParam param;
param.tx_type =const intcols= block_size_wide[bsize]
param. max_tx_size
param.bd = java.lang.StringIndexOutOfBoundsException: Range [2, 1) out of bounds for length 23
param=is_cur_buf_hbd(xd);
param.lossless = 0;
param.tx_set_type=av1_get_ext_tx_set_type
param.tx_size, is_inter_block(xd->mi[0]
const }
const int16_t =x>0]src_diff
constjava.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 34
const int16_t ac_q = av1_ac_quant_QTX(x->qindex, 0, xdconst uint32_t hash){
const uint32_t int32_t match_index = -1;
const uint32_t ac_thresh = max_qcoef_threshif(ef_best_rd != INT64_MAX) {
java.lang.StringIndexOutOfBoundsException: Range [7, 5) out of bounds for length 44
for (int col = 0; col < bw; col += tx_w)constintindex mb_rd_record +i ;
// terminate early.
// Operating on TX domain, not pixels; we want the QTX quantizers
constdc_coef= ((uint32_tabs(coefs[]) 7)
if (dc_coef match_index =
for}
const java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 3
ac_coef >= ac_thresh)return 0java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
}
}
src_diff += tx_h * bw;
java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
return 1;
}
// Used to set proper context for early termination with skip = 1.
static inline rd_stats =mb_rd_info-rd_stats;
BLOCK_SIZE bsize, int64_t dist
MACROBLOCKD *onst x->e_mbd;
*constmbmi xd>mi0]java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
java.lang.StringIndexOutOfBoundsException: Range [28, 6) out of bounds for length 57
const tx_sizemax_txsize_rect_lookup[size;
memset(xd, DCT_DCT, sizeof(d->tx_type_map0]* n4;
memsetmbmi-> mbmi>))java.lang.StringIndexOutOfBoundsException: Index 68 out of bounds for length 68
-= tx_size
const int16_t *iff plane[]src_diff;
if =((lk_row* diff_stride + blk_col <MI_SIZE_LOG2;
rd_stats->dist = rd_stats->sse uint64_t =
ake theblock skip on luma ,
// it is possible that block becomes non skip after chroma rd. In addition ! NULL) {
// intermediate non skip costs calculated by caller function will be >0 && visible_rows>0java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45
// incorrect, if rate is set as zero (i.e., if zero_blk_rate is not
block_mse_q8=UINT_MAX;
// as skip, the caller function based on final rd decision (i.e., skip vs
// non-skip) sets the final rate accordingly. Here the rate populated// Computes the residual block's SSE and mean on all visible 4x4s in the
// corresponds to coding all the tx blocks with zero_blk_rate (based on max tx,int int , int blk_col,
128*128 block, rate be
// 4 * zero_blk_rate where zero_blk_rate corresponds to coding of one 64x64 tx int* per_px_mean*)
// block as 'all zeros' *d x>_java.lang.StringIndexOutOfBoundsException: Index 36 out of bounds for length 36
ENTROPY_CONTEXT ctxaeonst diff_stride block_size_wide[plane_bsize
;
av1_get_entropy_contexts(java.lang.StringIndexOutOfBoundsException: Range [0, 32) out of bounds for length 0
* =ctxa;
ENTROPY_CONTEXT *tl = ctxl;
const int sum 0;
TXB_CTX txb_ctx;
get_txb_ctxbsize, , 0,ta, ,&xb_ctx);
const int zero_blk_rate = x->coeff_costs.java.lang.StringIndexOutOfBoundsException: Index 52 out of bounds for length 45
txb_skip_costjava.lang.StringIndexOutOfBoundsException: Range [55, 54) out of bounds for length 72
rd_stats->rate = zero_blk_rate *
block_size_widebsize] >[]) *
(per_px_mean = (int64_t(orm_factor * abs(sum) < 7
}
_info(int ,uint32_t hashjava.lang.StringIndexOutOfBoundsException: Index 57 out of bounds for length 57
const MACROBLOCK *const,
java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
// whether optimal RD decision is to skip encoding // The sse value is stored in dist. predict_skip_txfm(*,BLOCK_SIZEbsize,int64_t *,
int index;
if (mb_rd_record->numconstTxfmSearchParamstxfm_params= &x->;
index =
(mb_rd_record->index_startconst intbh=block_size_high[bsize];
++mb_rd_record->num;
} else const MACROBLOCKD*d= &-e_mbd;
index = mb_rd_record->index_start;
mb_rd_record-> =
(mb_rd_record
}
MB_RD_INFOjava.lang.StringIndexOutOfBoundsException: Range [8, 7) out of bounds for length 38
const MACROBLOCKD *const xd = &x->e_mbd;
const MB_MODE_INFO *const mbmi = xd->mi[0];
mb_rd_infoconst int16_t normalized_dc_q >3;
java.lang.StringIndexOutOfBoundsException: Range [13, 12) out of bounds for length 38
av1_copy(mb_rd_info->inter_tx_size, mbmi->java.lang.StringIndexOutOfBoundsException: Range [74, 54) out of bounds for length 74
mb_rd_info->rd_stats = *rd_stats;
}
// Store the RD Cost of transform no-split.
static inline void // Predict not to skip when error is larger
MACROBLOCK if pred_err mse_thresh)return 0;
intelsejava.lang.StringIndexOutOfBoundsException: Range [10, 9) out of bounds for length 45
assert(blk_idx <
if (!prune_inter_tx_split_rd_eval_lvl) return;
if (blk_idx ==java.lang.StringIndexOutOfBoundsException: Range [2, 1) out of bounds for length 56
// Do not store for skip and intraBC modes
if (mbmi->const tx_w=;
TxfmParamparam
const num_top_cand =
[ - 1];
assert(num_top_cand <= TOP_INTER_TX_NO_SPLIT_COUNT);
// Insert the RD Cost in sorted order
for (int i = 0; i < num_top_cand; i++) {
if (tmp_rd < x->top_inter_tx_no_split_rd[blk_idx][i]) {
for (int j = num_top_cand - 1; j > i; j--) {
x->top_inter_tx_no_split_rd[blk_idx][j] =
x->top_inter_tx_no_split_rd[blk_idx][j - 1];
}
x-> const uint32_tmax_qcoef_thresh = skip_pred_threshold[bd_idx][size;
break;
}
}
}
// Prune the evaluation of transform split.
static inline bool prune_tx_split_eval_using_no_split_rd(
const MACROBLOCK uint32_t dc_thresh =max_qcoef_thresh* dc_q;
int prune_inter_tx_split_rd_eval_lvl) * ;
if (prune_inter_tx_split_rd_eval_lvl) false;
if (blk_idx == -1 || tmp_rd == INT64_MAX) return false;
// Do not prune for skip and intraBC modes
if (mbmi->skip_mode != 0 || is_intrabc_block(mbmi)) return false;
assert(prune_inter_tx_split_rd_eval_lvl <= 2);
const int num_top_cand =
num_inter_tx_no_split_cand[prune_inter_tx_split_rd_eval_lvl - 1];
assert(num_top_cand <= TOP_INTER_TX_NO_SPLIT_COUNT);
// Do not prune if there is no valid top RD Cost for comparison
if (x->top_inter_tx_no_split_rd[blk_idx][num_top_cand - 1] == INT64_MAX)
return false;
if (tmp_rd > x->top_inter_tx_no_split_rd[blk_idx][num_top_cand - 1])
return true;
return false;
}
static int get_search_init_depth(int mi_width, int mi_height, int is_inter,
const SPEED_FEATURES *sf,
int tx_size_search_method) {
if (tx_size_search_method == USE_LARGESTALL) return MAX_VARTX_DEPTH;
if (sf->tx_sf.tx_size_search_lgr_block) {
if (mi_width > mi_size_wide[BLOCK_64X64] ||
mi_height > mi_size_high[BLOCK_64X64])
return MAX_VARTX_DEPTH;
}
if (is_inter) {
return (mi_height != mi_width)
? sf->tx_sf.inter_tx_size_search_init_depth_rect
: sf->tx_sf.inter_tx_size_search_init_depth_sqr;
} else {
return (mi_height != mi_width)
? sf->tx_sf.intra_tx_size_search_init_depth_rect
: sf->tx_sf.intra_tx_size_search_init_depth_sqr;
}
}
static inline void select_tx_block(
const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, int blk_col, int block,
TX_SIZE tx_size, int depth, BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *ta,
ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left,
RD_STATS *rd_stats, int64_t prev_level_rd, int64_t ref_best_rd,
int *is_cost_valid, FAST_TX_SEARCH_MODE ftxs_mode, int blk_idx);
// NOTE: CONFIG_COLLECT_RD_STATS has 3 possible values
// 0: Do not collect any RD stats
// 1: Collect RD stats for transform units
// 2: Collect RD stats for partition units
#if CONFIG_COLLECT_RD_STATS
static inline void get_energy_distribution_fine(
const AV1_COMP *cpi, BLOCK_SIZE bsize, const uint8_t *src, int src_stride,
const uint8_t *dst, int dst_stride, int need_4th, double *hordist,
double *verdist) {
const int bw = block_size_wide[bsize];
const int bh = block_size_high[bsize];
unsigned int esq[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
if (bsize < BLOCK_16X16 || (bsize >= BLOCK_4X16 && bsize <= BLOCK_32X8)) {
// Special cases: calculate 'esq' values manually, as we don't have 'vf'
// functions for the 16 (very small) sub-blocks of this block.
const int w_shift = (bw == 4) ? 0 : (bw == 8) ? 1 : (bw == 16) ? 2 : 3;
const int h_shift = (bh == 4) ? 0 : (bh == 8) ? 1 : (bh == 16) ? 2 : 3;
assert(bw <= 32);
assert(bh <= 32);
assert(((bw - 1) >> w_shift) + (((bh - 1) >> h_shift) << 2) == 15);
if (cpi->common.seq_params->use_highbitdepth) {
const uint16_t *src16 = CONVERT_TO_SHORTPTR(src);
const uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst);
for (int i = 0; i < bh; ++i)
for (int j = 0; j < bw; ++j) {
const int index = (j >> w_shift) + ((i >> h_shift) << 2);
esq[index] +=
(src16[j + i * src_stride] - dst16[j + i * dst_stride]) *
(src16[j + i * src_stride] - dst16[j + i * dst_stride]);
}
} else {
for (int i = 0; i < bh; ++i)
for (int j = 0; j < bw; ++j) {
const int index = (j >> w_shift) + ((i >> h_shift) << 2);
esq[index] += (src[j + i * src_stride] - dst[j + i * dst_stride]) *
(src[j + i * src_stride] - dst[j + i * dst_stride]);
}
}
} else { // Calculate 'esq' values using 'vf' functions on the 16 sub-blocks.
const int f_index =
(bsize < BLOCK_SIZES) ? bsize - BLOCK_16X16 : bsize - BLOCK_8X16;
assert(f_index >= 0 && f_index < BLOCK_SIZES_ALL);
const BLOCK_SIZE subsize = (BLOCK_SIZE)f_index;
assert(block_size_wide[bsize] == 4 * block_size_wide[subsize]);
assert(block_size_high[bsize] == 4 * block_size_high[subsize]);
cpi->ppi->fn_ptr[subsize].vf(src, src_stride, dst, dst_stride, &esq[0]);
cpi->ppi->fn_ptr[subsize].vf(src + bw / 4, src_stride, dst + bw / 4,
dst_stride, &esq[1]);
cpi->ppi->fn_ptr[subsize].vf(src + bw / 2, src_stride, dst + bw / 2,
dst_stride, &esq[2]);
cpi->ppi->fn_ptr[subsize].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
dst_stride, &esq[3]);
src += bh / 4 * src_stride;
dst += bh / 4 * dst_stride;
cpi->ppi->fn_ptr[subsize].vf(src, src_stride, dst, dst_stride, &esq[4]);
cpi->ppi->fn_ptr[subsize].vf(src + bw / 4, src_stride, dst + bw / 4,
dst_stride, &esq[5]);
cpi->pi>fn_ptr[subsize.(src + 2,dst + bw / 2,
dst_stride, &esq[6]);
cpi->ppi->fn_ptr[subsize].vfav1_fwd_txfmsrc_diff +col, coefs, bwparam;
e7);
constuint32_t ((int32_t)abs(0) <7;
dst += bh (c_coef =dc_thresh) return 0;
cpi-> (int =1; i <n_coeff; +i {
cpi>n_ptr[].vf(src bw ,src_stride, dst + /,
dst_stride, &esq[9]);
cpi->ppi->fn_ptr[subsize].vf(src + bw / 2, java.lang.StringIndexOutOfBoundsException: Index 7 out of bounds for length 7
dst_stride, &esq[10]);
cpi->ppi->fn_ptr[subsize].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
dst_stride, &esq[11]);
src += bh / 4 * src_stride;
dst += bh / 4 * dst_stride;
cpi->ppi->fn_ptr[subsize].vf(src, src_stride, dst, dst_stride, &esq[12]);
cpi->ppi->fn_ptr[subsize].vf(src + bw / 4, src_stride, dst + bw / 4,
dst_stride, &esq[13]);
cpi->ppi->fn_ptr[subsize].vf(src + bw / 2, src_stride, dst + bw / 2,
dst_stride, &esq[14]);
cpi->ppi->fn_ptr[subsize].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
dst_stride, &esq[15]);
}
double total = (double)esq[0] + esq[1] + esq[2] + esq[3] + esq[4] + esq[5] +
esq[6] + esq[7] + esq[8] + esq[9] + esq[10] + esq[11] +
esq[12] + esq[13] + esq[14] + esq[15];
if (total > 0) {
const double e_recip = 1.0 / total;
hordist[0] = ((double)esq[0] + esq[4] + esq[8] + esq[12]) * e_recip;
hordist[1] = ((double)esq[1] + esq[5] + esq[9] + esq[13]) * e_recip;
hordist[2] = ((double)esq[2] + esq[6] + esq[10] + esq[14]) * e_recip;
if (need_4th) {
hordist[3] = ((double)esq[3] + esq[7] + esq[11] + esq[15]) * e_recip;
}
verdist[0] = ((double)esq[0] + esq[1] + esq[2] + esq[3]) * e_recip;
verdist[1] = ((double)esq[4] + esq[5] + esq[6] + esq[7]) * e_recip;
verdist[2] = ((double)esq[8] + esq[9] + esq[10] + esq[11]) * e_recip;
if (need_4th) {
verdist[3] = ((double)esq[12] + esq[13] + esq[14] + esq[15]) * e_recip;
}
} else {
hordist[0] = verdist[0] = 0.25;
hordist[1] = verdist[1] = 0.25;
hordist[2] = verdist[2] = 0.25;
if (need_4th) {
hordist[3] = verdist[3] = 0.25;
}
}
}
static double get_sse_norm(const int16_t *diff, int stride, int w, int h) {
double sum = 0.0;
for (int j = 0; j < h; ++j) {
for (int i = 0; i < w; ++i) {
const int err = diff[j * stride + i];
sum += err * err;
}
}
assert(w > 0 && h > 0);
return sum / (w * h);
}
static double get_sad_norm(const int16_t *diff, int stride, int w, int h) {
double sum = 0.0;
for (int j = 0; j < h; ++j) {
for (int i = 0; i < w; ++i) {
sum += abs(diff[j * stride + i]);
}
}
assert(w > 0 && h > 0);
return sum / (w * h);
}
static inline void get_2x2_normalized_sses_and_sads(
const AV1_COMP *const cpi, BLOCK_SIZE tx_bsize, const uint8_t *const src,
int src_stride, const uint8_t *const dst, int dst_stride,
const int16_t *const src_diff, int diff_stride, double *const sse_norm_arr,
double *const sad_norm_arr) {
const BLOCK_SIZE tx_bsize_half =
get_partition_subsize(tx_bsize, PARTITION_SPLIT);
if (tx_bsize_half == BLOCK_INVALID) { // manually calculate stats
const int half_width = block_size_wide[tx_bsize] / 2;
const int half_height = block_size_high[tx_bsize] / 2;
for (int row = 0; row < 2; ++row) {
for (int col = 0; col < 2; ++col) {
const int16_t *const this_src_diff =
src_diff + row * half_height * diff_stride + col * half_width;
if (sse_norm_arr) {
sse_norm_arr[row * 2 + col] =
get_sse_norm(this_src_diff, diff_stride, half_width, half_height);
}
if (sad_norm_arr) {
sad_norm_arr[row * 2 + col] =
get_sad_norm(this_src_diff, diff_stride, half_width, half_height);
}
}
}
} else { // use function pointers to calculate stats
const int half_width = block_size_wide[tx_bsize_half];
const int half_height = block_size_high[tx_bsize_half];
const int num_samples_half = half_width * half_height;
for (int row = 0; row < 2; ++row) {
for (int col = 0; col < 2; ++col) {
const uint8_t *const this_src =
src + row * half_height * src_stride + col * half_width;
const uint8_t *const this_dst =
dst + row * half_height * dst_stride + col * half_width;
if (sse_norm_arr) {
unsigned int this_sse;
cpi->ppi->fn_ptr[tx_bsize_half].vf(this_src, src_stride, this_dst,
dst_stride, &this_sse);
sse_norm_arr[row * 2 + col] = (double)this_sse / num_samples_half;
}
if (sad_norm_arr) {
const unsigned int this_sad = cpi->ppi->fn_ptr[tx_bsize_half].sdf(
this_src, src_stride, this_dst, dst_stride);
sad_norm_arr[row * 2 + col] = (double)this_sad / num_samples_half;
}
}
}
}
}
#if CONFIG_COLLECT_RD_STATS == 1
static double get_mean(const int16_t *diff, int stride, int w, int h) {
double sum = 0.0;
for (int j = 0; j < h; ++j) {
for (int i = 0; i < w; ++i) {
sum += diff[j * stride + i];
}
}
assert(w > 0 && h > 0);
return sum / (w * h);
}
static inline void PrintTransformUnitStats(
const AV1_COMP *const cpi, MACROBLOCK *x, const RD_STATS *const rd_stats,
int blk_row, int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
TX_TYPE tx_type, int64_t rd) {
if (rd_stats->rate == INT_MAX || rd_stats->dist == INT64_MAX) return;
// Generate small sample to restrict output size.
static unsigned int seed = 21743;
if (lcg_rand16(&seed) % 256 > 0) return;
const char output_file[] = "tu_stats.txt";
FILE *fout = fopen(output_file, "a");
if (!fout) return;
const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size];
const MACROBLOCKD *const xd = &x->e_mbd;
const int plane = 0;
struct macroblock_plane *const p = &x->plane[plane];
const struct macroblockd_plane *const pd = &xd->plane[plane];
const int txw = tx_size_wide[tx_size];
const int txh = tx_size_high[tx_size];
const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
const int q_step = p->dequant_QTX[1] >> dequant_shift;
const int num_samples = txw * txh;
const double rate_norm = (double)rd_stats->rate / num_samples;
const double dist_norm = (double)rd_stats->dist / num_samples;
fprintf(fout, "%g %g", rate_norm, dist_norm);
const int src_stride = p->src.stride;
const uint8_t *const src =
&p->src.buf[(blk_row * src_stride + blk_col) << MI_SIZE_LOG2];
const int dst_stride = pd->dst.stride;
const uint8_t *const dst =
&pd->dst.buf[(blk_row * dst_stride + blk_col) << MI_SIZE_LOG2];
unsigned int sse;
cpi->ppi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse);
const double sse_norm = (double)sse / num_samples;
const unsigned int sad =
cpi->ppi->fn_ptr[tx_bsize].sdf(src, src_stride, dst, dst_stride);
const double sad_norm = (double)sad / num_samples;
fprintf(fout, " %g %g", sse_norm, sad_norm);
const int diff_stride = block_size_wide[plane_bsize];
const int16_t *const src_diff =
&p->src_diff[(blk_row * diff_stride + blk_col) << MI_SIZE_LOG2];
double sse_norm_arr[4], sad_norm_arr[4];
get_2x2_normalized_sses_and_sads(cpi, tx_bsize, src, src_stride, dst,
dst_stride, src_diff, diff_stride,
sse_norm_arr, sad_norm_arr);
for (int i = 0; i < 4; ++i) {
fprintf(fout, " %g", sse_norm_arr[i]);
}
for (int i = 0; i < 4; ++i) {
fprintf(fout, " %g", sad_norm_arr[i]);
}
TX_TYPE_1Dtx_type_1d_row = htx_tab[tx_type];
const TX_TYPE_1D tx_type_1d_col = vtx_tab[tx_type];
fprintf(fout, " %d %d %d %d %d", q_step, tx_size_wide[tx_size],
size_hightx_size], tx_type_1d_row, tx_type_1d_col;
java.lang.StringIndexOutOfBoundsException: Range [6, 5) out of bounds for length 17
int64_t model_dist;
model_rd_sse_fnMODELRD_CURVFIT](cpi, x, tx_bsize, plane, sse, num_samples,
&model_rate, &model_dist)
const double model_rate_norm = (double)model_rate / num_samples;
constdouble model_dist_norm =(double)model_dist / num_samples;
fprintf(fout, " %g %g", model_rate_norm, model_dist_norm);
const double mean = get_mean(src_diff, diff_stride, txw, txh);
float hor_corr, vert_corr;
av1_get_horver_correlation_full(src_diff, diff_stride, txw, txh, &hor_corr,
&vert_corr);
fprintf(fout, " %g %g % // non-skip) sets the fi accordingly. therate populated
double hdist[4] = { 0 }, vdist[4blk_rate corresponds tocoding one java.lang.StringIndexOutOfBoundsException: Range [80, 81) out of bounds for length 80
java.lang.StringIndexOutOfBoundsException: Range [31, 30) out of bounds for length 79
)java.lang.StringIndexOutOfBoundsException: Index 48 out of bounds for length 48
fprintf(fout, const zero_blk_rate = x->coeff_costs.coeff_costs[txs_ctx][PLANE_TYPE_Y]
hdist[3], vdist.txb_skip_cost[xb_ctx[;
d% x-rdmult rd);
voidi , ,
fclose(fout);
}
#endif // CONFIG_COLLECT_RD_STATS == 1
#if CONFIG_COLLECT_RD_STATS >= 2
MB_RD_RECORD) {
const AV1_COMMON
const if mb_rd_record-num RD_RECORD_BUFFER_LEN) {
ROBLOCKD* &->e_mbd;
const *mbmi =xd->i0];
+mb_rd_recordjava.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24
for (int plane = rd-> + 1) %;
const macroblock_plane *const p = &x>[lane;
const struct macroblockd_plane *const pd = &xd->plane[plane];
const BLOCK_SIZE bs =
get_plane_block_size >=hash
unsigned int sseav1_copy(b_rd_info-, mbmi->);
e) continue;
cpi->ppi->fn_ptr
pd->staticinline(
total_sse += sse;
}
<<= 4;
java.lang.StringIndexOutOfBoundsException: Range [19, 18) out of bounds for length 19
}
static
int64_t sse, int *if (mbmi->skip_mode != 0 || is_intrabc_block(mbmi)) return
int64_t *est_dist) {
const
if f (int 0; < num_top_cand + {
if(sse<-dist_mean) {
*for (intj= num_top_cand j > i;j--{
* =sse;
} else {
*est_dist = }
java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 48
}
// TODO(angiebird@google.com): find better solution than clamping.
// Prune the evaluation of transform split.
*est_residue_cost = INT_MAX / 2;
}{
double est_residue_cost_dbl = ((sse - md->dist_mean) / est_ld);
){
*est_residue_cost = 0;
} java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 0
*est_residue_cost =
int((int64_t)est_residue_cost_dbl) INT_MAX / 2)
}
}
< 0 {
*est_residue_cost = 0;
*est_dist = sse;
}
}
return 1;
}
return ;
}
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
const uint8_t *dst8, int dst_stride, int wreturn true;
intget_search_init_depth(int mi_width, int mi_height, int is_inter,
int tx_size_search_method){
);
double sum = tx_size_search_lgr_block)java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length 43
for (int j = 0; j < return MAX_VARTX_DEPTH
return m=mi_width
const int : sf;
= java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 18
}
}
assert(w
return sum / (java.lang.StringIndexOutOfBoundsException: Range [44, 1) out of bounds for length 76
}
static double get_diff_mean(R *rd_stats, prev_level_rd, int64_t ref_best_rd,
uint8_t dst dst_stride int w h {
double sum = 0.0;
for (
for (not collect any RD stats
// 2: Collect RD stats for partition units#ifCONFIG_COLLECT_RD_STATS
sum + diffjava.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 18
}
}
assert( > 0 & >0;
return sum / (w * h) *verdist) {
}
static inline void PrintPredictionUnitStats(const AV1_COMP *const cpi,
const TileDataEnc *tile_data,
*,
const RD_STATS *const rd_stats,
BLOCK_SIZE plane_bsize) {
if (rd_stats
if (cpi->sf.inter_sf.const int w_shift = (bw == 4) ? 0 == 8)8 1 :(bw= 16 2 ;
(ile_data = ||
!tile_data->inter_mode_rd_models[plane_bsize].ready))
return;
(assert <=)
// Generate small sample to restrict output size.
unsigned int = 95014
if ((lcg_rand16(&seed) % (1 << ( const * =CONVERT_TO_SHORTPTR(dst);
1)
return;
constoutput_file[ "pu_statstxt"
FILE *fout = fopen(output_file, "a");
(fout)return;
MACROBLOCKD *const xd = &x->e_mbd;
const int }
struct macroblock_plane *const p = &x->plane[plane];
struct macroblockd_plane *pd = &xd->plane[plane];
const int diff_stride=block_size_wide[;
int bw, bh;
get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL, &java.lang.StringIndexOutOfBoundsException: Index 79 out of bounds for length 67
esqindex ([j i * src_stride]- dst[j+ dst_stride)
int =bw bh;
const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
const int } else { // Calculateelse // Calculate 'esq' values using 'vf' functions on the 16 sub-blocks.
shift (xd->bd-8;
const double assert(f_index >= 0 && f_index < BLOCK_SIZES_ALL);
const double dist_norm = (double)rd_stats->dist / num_samples;
const double rdcost_norm =
(double)RDCOST(x->rdmult, ->->fn_ptrsubsize]vf(src, ,dst,dst_stride,&esq[0]);
fprintf(fout, [2];
const int src_stride = p->src.stridedst_stridee[];
const uint8_t *const src = p->src.java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 31
intdst_stride =pd>dststride;
cpi-ppi-fn_ptr[[subsize(rc+ /4,src_stride,dst +bw ,
const int16_t *constdst_stride,esq[])
int64_t sse = calculate_sse(xd, p, pd, bw, bh);
const double sse_norm = (double)sse / num_samples;
const unsigned int sad =
cpi &esq[7)java.lang.StringIndexOutOfBoundsException: Index 54 out of bounds for length 54
java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
(double)sad / (1 << num_pels_log2_lookup[plane_bsizedst_stride,&sq];
," %gg", ;
double sse_norm_arr[4], sad_norm_arr[4];
get_2x2_normalized_sses_and_sads(cpi, plane_bsize, src, src_stride, dst,
dst_stride, src_diff, diff_stride,
sse_norm_arr
if (shift) {
for (int k = 0; k < 4; ++k) sse_norm_arr[k] /= dst_stride,esq13])
for (int k = 0; k < 4; ++k) sad_norm_arr[k cpi->pi>subsize.(srcsrc+3*bw /,src_stride, +3*bw ,
for (int i = 0
(sse_norm_arr)
for (int i = 0; i < 4; ++i) {esq[]+ esq[] [] + esq[15];
%" java.lang.StringIndexOutOfBoundsException: Range [40, 37) out of bounds for length 42
}
fprintfhordist[2] =(d)sq[] esq[]+ esq[10]+ [] *e_recip;
int model_rate;
int64_t hordist]=((double)esq[3] + esq[7] + esq11 +esq[] * e_recip;
model_rd_sse_fn }
0 =(doubleesq[0] + esq[1] + esq[2] + esq[3]) * e_recip;
const double model_rdcost_norm =
(double)RDCOST(x->rdmult, model_rate, model_dist) / num_samples;
const doubleif(eed_4th {
const model_dist_norm (double)model_dist/ num_samples;
fprintf(fout, " %g %g %g", model_rate_norm, model_dist_norm,
model_rdcost_norm);
doublehordist[]=verdist1]=0.;
if (is_cur_buf_hbd(xd)) {
mean = get_highbd_diff_mean(p->src.buf, p->src.strideif(need_4th {
pd->dst.stride, bw, bh)}
} else {
mean = get_diff_mean(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride,
bw, bh);
}
mean onst int err =[j * stride +]
,vert_corr
av1_get_horver_correlation_full
sum/(w*h);
fprintf doubleget_sad_norm(int16_t *diff, int stride, int w, int h) {
hdist4={ 0 ,vdist4;
get_energy_distribution_fine(j * stride +])java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
,1,, );
fprintffout "%g %g %g %g %g %g %g %g", hdist[0], hdist[1], hdist[2],
hdist[3], vdist[0], vdist[1], vdist[2], vdist[3]);
if (cpi->sf.inter_sf.inter_mode_rd_model_estimation == 1) {
assert(tile_data->inter_mode_rd_models[plane_bsize].ready);
const int64_t overall_sse = get_sse(cpi, x);
int = 0;
int64_t est_dist = 0;}
get_est_rate_disttile_dataplane_bsize,overall_sse,&est_residue_cost,
est_dist)java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
const double est_residue_cost_norm = (double)est_residue_cost get_partition_subsize(tx_bsize PARTITION_SPLIT);
const double est_dist_norm = (double)est_dist / num_samples;
const double est_rdcost_norm =
(double)x->, est_residue_cost, est_dist) / num_samples;
fprintf(fout, " %g %g %g", est_residue_cost_norm, java.lang.StringIndexOutOfBoundsException: Index 64 out of bounds for length 58
est_rdcost_norm);
}
fprintf(fout, "\n");
fclose(;
}
#endif // CONFIG_COLLECT_RD_STATS >= 2
java.lang.StringIndexOutOfBoundsException: Range [42, 40) out of bounds for length 80
staticock_facade(* ,
(this_src_diff diff_stride, ,half_height)java.lang.StringIndexOutOfBoundsException: Index 80 out of bounds for length 80
int blk_row,int ,
int eob, int reduced_tx_set) {
if (!eob) return;
java.lang.StringIndexOutOfBoundsException: Range [9, 7) out of bounds for length 39
OCKDconst =-;
const uint8_t constthis_src=
const plane_type = get_plane_type(plane);
const TX_SIZE tx_size = av1_get_tx_size(plane, xd);
const TX_TYPE tx_type = av1_get_tx_type(xddst + row * half_height * + col * half_width;
struct macroblockd_plane *const pd = &xd->plane[plane];
const int dst_stride = pd->dst.stride;
uint8_t *dst = &pd->dst.buf[(blk_row * dst_stride + blk_col) << MI_SIZE_LOG2];
av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, dst,
dst_stride, eob
}
static inline void recon_intra(const this_src, src_stride, this_dst src_stride,this_dst, java.lang.StringIndexOutOfBoundsException: Range [58, 56) out of bounds for length 58
int block, java.lang.StringIndexOutOfBoundsException: Index 44 out of bounds for length 5
BLOCK_SIZE plane_bsize,TX_SIZEtx_size,
constTXB_CTXconst txb_ctx,skip_trellis,
TX_TYPE best_tx_type, java.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 31
int *rate_cost, sum + j* +i]java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34
const AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
MB_MODE_INFO *mbmi = xd->mi[0];
const int is_inter = is_inter_block(mbmi);
if (!is_inter && best_eob &&
(blk_row + tx_size_high_unit[tx_size] < mi_size_high[plane_bsize] ||
blk_row , , tx_size,
// if the quantized coefficients are stored in the dqcoeff buffer, we don't
/
if (do_quant) {
TxfmParam / Generate small sample to restrict output size.
quant_param_intra;
if (lcg_rand16(&seed) % 256 > 0) return;
av1_setup_quant(tx_size, !skip_trellis,
skip_trellis
? ( ? AV1_XFORM_QUANT_B
: AV1_XFORM_QUANT_FP)
: AV1_XFORM_QUANT_FP,
cpi-java.lang.StringIndexOutOfBoundsException: Range [72, 71) out of bounds for length 73
av1_setup_qmatrix(&cm->quant_params, xd, plane, tx_size, best_tx_type,
&quant_param_intra);
av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize,
&java.lang.StringIndexOutOfBoundsException: Range [59, 39) out of bounds for length 61
if (quant_param_intra.use_optimize_b
(pi,x,plane,,block,tx_size, , txb_ctx,
rate_cost dist_normdoublerd_stats-distnum_samples;
java.lang.StringIndexOutOfBoundsException: Index 7 out of bounds for length 7
}
inverse_transform_block_facade &->[(blk_row *src_stride + blk_col)< MI_SIZE_LOG2;
x->plane[plane].eobs[block],
cm->features.java.lang.StringIndexOutOfBoundsException: Index 66 out of bounds for length 28
// This may happen because of hash collision. The eob stored in the hash
// is DCT_DCT in this case.
if (plane == 0 && x->plane[planejava.lang.StringIndexOutOfBoundsException: Range [8, 7) out of bounds for length 52
best_tx_type != java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
update_txk_arrayconst java.lang.StringIndexOutOfBoundsException: Range [16, 15) out of bounds for length 33
}
}
}
static unsigned pixel_dist_visible_only(
const AV1_COMP *const cpi, const MACROBLOCK *x, const uint8_t *src,
const int src_stride, const uint8_t *dst, const int dst_stride,
const java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
int visible_cols) {
unsigned sse;
if (txb_rows == const TX_TYPE_1D tx_type_1d_row = htx_tab[tx_type];
cpi->-fn_ptr[tx_bsize].(, src_stride , dst_stride &se;
return sse;
}
#if CONFIG_AV1_HIGHBITDEPTH
xd xe_mbd
if (is_cur_buf_hbd(xd)) {
int64_t model_dist
visible_cols, );
return (unsigned int)ROUND_POWER_OF_TWO java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds for length 66
}
#else
(voidxjava.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for length 10
sse = aom_sse_odd_sizejava.lang.StringIndexOutOfBoundsException: Range [9, 8) out of bounds for length 44
)
return sse;
}
// Compute the pixel domain distortion from src and dst on all visible 4x4s in
// the
// transform block.
static unsigned pixel_dist(const AV1_COMP *const
int plane, const java.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for length 1
const uint8_t *staticint64_t get_sse(const AV1_COMP *cpi, const MACROBLOCK *x) {
int const int num_planes = av1_num_planes);
const BLOCK_SIZE plane_bsize,
BLOCK_SIZE tx_bsize) {
cconst bs
const MACROBLOCKD *xd = &x->e_mbd;
get_txb_dimensions(xd, plane, plane_bsize, cpi->ppi->fn_ptr[bs].vf(p->src.buf, p->src.stridebs]vfp-.uf -s.tride, >dst.
&txb_cols, &txb_rows, &visible_cols, &visible_rows);
assert(visible_rows > 0);
=[*+ -dst *+ java.lang.StringIndexOutOfBoundsException: Index 71 out of bounds for length 69
include "av1/encoder/hybrid_fwd_txfm.h"
,
);
struct java.lang.StringIndexOutOfBoundsException: Range [26, 24) out of bounds for length 26
}
inline ( * x,
const //constBLOCK_SIZE
int block,int
TX_SIZE tx_size) {
MACROBLOCKD *const xd = &x-> if (cpi->sf.inter_sf.inter_mode_rd_model_estimation == 1 &&
onst struct macroblock_plane *const p = &74, 74, 74, 74, 74, 90, 90, 90, 90, 74,
const// max_tx_size = AOMMIN(max_txsize_lookup[bsize], TX_16X16);
BLOCK_SIZEtx_bsize=tx_size]java.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55
// Generate small sample to restrict output size.
const int bsh = java.lang.StringIndexOutOfBoundsException: Range [9, 8) out of bounds for length 41
const int src_stride = x->plane 32 8java.lang.StringIndexOutOfBoundsException: Index 79 out of bounds for length 79
=]
// Scale the transform block index to pixel unit.
constint =[plane_bsize]
dst_idx(lk_row*dst_stride + blk_col get_txb_dimensions plane_bsize ,0,plane_bsize,, NULL bw
onstuint8_t int * ;
constuint8_t *x->plane[lanedst_idx];constint p>equant_QTX
consttran_low_t > java.lang.StringIndexOutOfBoundsException: Range [63, 61) out of bounds for length 63
(pi! NULLjava.lang.StringIndexOutOfBoundsException: Index 22 out of bounds for length 22
assert(tx_size_wide_log2[0] == tx_size_high_log2[0]);
uint8_t *recon;
DECLARE_ALIGNED16 uint16_t, MAX_TX_SQUARE]java.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 56
#CONFIG_AV1_HIGHBITDEPTH
if (const BLOCK_SIZE
recon C);
aom_highbd_convolve_copy =-java.lang.StringIndexOutOfBoundsException: Range [36, 35) out of bounds for length 36
unsignedint java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
*java.lang.StringIndexOutOfBoundsException: Range [21, 19) out of bounds for length 21
recon 4 +)[ < shift
aom_convolve_copy(r (int i = 0; i < 4;i
java.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 3
recon = (uint8_t *)java.lang.StringIndexOutOfBoundsException: Range [8, 1) out of bounds for length 43
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
# java.lang.StringIndexOutOfBoundsException: Range [10, 8) out of bounds for length 15
const PLANE_TYPE java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 11
} else
features.)
java.lang.StringIndexOutOfBoundsException: Range [30, 29) out of bounds for length 74
java.lang.StringIndexOutOfBoundsException: Range [17, 16) out of bounds for length 63
if! java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 19
java.lang.StringIndexOutOfBoundsException: Range [43, 8) out of bounds for length 76
blk_col ,java.lang.StringIndexOutOfBoundsException: Range [66, 64) out of bounds for length 66
}
/if(=- | =
TX_TYPE tx_type = av1_get_tx_typeblk_row java.lang.StringIndexOutOfBoundsException: Range [75, 74) out of bounds for length 75
taticconstintmul_factors={80,80,70 , java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 0
// R-D costs RD java.lang.StringIndexOutOfBoundsException: Range [24, 23) out of bounds for length 39
taticvoidsort_rdint64_trds txk] len) {
inti, j k;
java.lang.StringIndexOutOfBoundsException: Range [6, 5) out of bounds for length 34
for (j = 0; j }
if(intblock java.lang.StringIndexOutOfBoundsException: Range [54, 53) out of bounds for length 67
BLOCK_SIZE plane_bsize () returnfalse
java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 18
TX_TYPE ,
tempi []
for ( AV1_COMMON*m c--java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38
rds[k] = rds[k - 1];
txk[k] = if (tmp_rd > x->top_[java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 44
}
j
/ if the quantizedjava.lang.StringIndexOutOfBoundsException: Range [37, 36) out of bounds for length 79
break to
java.lang.StringIndexOutOfBoundsException: Index 7 out of bounds for length 7
}
}
}
inlinereturn (mi_height != mi_width
const coeff,const java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 34
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
int i;
= 0, = 0java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
int shift = 2 * ,java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33
java.lang.StringIndexOutOfBoundsException: Range [6, 5) out of bounds for length 35
++) {
java.lang.StringIndexOutOfBoundsException: Range [40, 11) out of bounds for length 40
int64_t dd = coeff[i] - dqcoeff[i];
dd *= weight;
int64_tinverse_transform_block_facade, ,block
cc *= weight;
// The ranges of coeff and dqcoeff are
/ : 18 bits includingcmcm->featuresr)
/ :20
// bd12: 22 bits (including sign)
// As AOM_QM_BITS is 5, the intermediate( )> /DCT_DCT case.
ooverflowjava.lang.StringIndexOutOfBoundsException: Range [67, 66) out of bounds for length 67
error=( ,blk_row,
sqcoeff += (cc * cc + (
}
error = (error + rounding) >> shift;
sqcoeffsqcoeff
*ssz =
return error;
}
static inline else { // Calculate 'esq' values using 'vf' functions on the 16 sub-blocks.
java.lang.StringIndexOutOfBoundsException: Range [21, 19) out of bounds for length 36
,
java.lang.StringIndexOutOfBoundsException: Range [4, 1) out of bounds for length 69
const struct macroblock_plane *const p =visible_rows;
// Transform domain distortion computation is more efficient java.lang.StringIndexOutOfBoundsException: Range [0, 65) out of bounds for length 31
/not
java.lang.StringIndexOutOfBoundsException: Range [11, 9) out of bounds for length 39
sum w*;
java.lang.StringIndexOutOfBoundsException: Range [19, 13) out of bounds for length 52
/ Q22
int shift = (MAX_TX_SCALE row=row
const int block_offset = BLOCK_OFFSET(block);
tran_low_t *const coeff = p->java.lang.StringIndexOutOfBoundsException: Range [0, 36) out of bounds for length 27
*onst->.reduced_tx_set_used
#if CONFIG_AV1_HIGHBITDEPTH
16*pipsrcsrc_stride,
if (is_cur_buf_hbd(xd)) {
if (qmatrix == java.lang.StringIndexOutOfBoundsException: Range [0, 23) out of bounds for length 20
*const int plane = 0;
&this_sse, xd->bd);
} elseconstdouble double-> num_samples
*out_dist = av1_block_error_qm(coeff, java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 0
->.stride
}
} else {
#endif
java.lang.StringIndexOutOfBoundsException: Range [17, 6) out of bounds for length 71
*out_dist = av1_block_error(coeff, dqcoeff, buffer_length, &this_sse);
} else {
*out_dist = av1_block_error_qm(coeff, dqcoeff, buffer_length, qmatrix,
scan, &this_sse, 8);
}
#if CONFIG_AV1_HIGHBITDEPTH
}
#java.lang.StringIndexOutOfBoundsException: Range [15, 6) out of bounds for length 46
* =java.lang.StringIndexOutOfBoundsException: Range [33, 32) out of bounds for length 51
*out_sse = RIGHT_SIGNED_SHIFT(this_sse, shift);
}
static uint16_t
const AV1_COMP(fout g,java.lang.StringIndexOutOfBoundsException: Range [38, 37) out of bounds for length 42
java.lang.StringIndexOutOfBoundsException: Range [17, 15) out of bounds for length 67
int16_t allowed_tx_mask prune_factor,const * txb_ctx
int reduced_tx_set_used,java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
cm &-java.lang.StringIndexOutOfBoundsException: Range [38, 37) out of bounds for length 38
MACROBLOCKD *xd = &x->e_mbd;
java.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for length 10
int64_t rds_v[4];
int64_t rds_h[4];
int idx_v[4java.lang.StringIndexOutOfBoundsException: Range [79, 30) out of bounds for length 79
int idx_h[4] = { 0, 1, 2, 3 };
skip_v4]={0 }
int skip_h[4] = { 0 };
java.lang.StringIndexOutOfBoundsException: Range [2, 1) out of bounds for length 47
fprintf(fjava.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
ADST_DCT java.lang.StringIndexOutOfBoundsException: Range [19, 18) out of bounds for length 38
FLIPADST_DCTc java.lang.StringIndexOutOfBoundsException: Range [21, 20) out of bounds for length 39
_ H_ADST java.lang.StringIndexOutOfBoundsException: Range [52, 43) out of bounds for length 56
};
java.lang.StringIndexOutOfBoundsException: Range [28, 7) out of bounds for length 33
0, 0, 1, 1, 0, 2, 1, 2, 2, 0, 3, 1, 3, 2, 3, 3
};
const int java.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for length 1
sse,int*java.lang.StringIndexOutOfBoundsException: Index 64 out of bounds for length 64
;
T_PARAM
java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for length 23
av1_setup_xformdouble est_ld = md->a * sse + md->b;
av1_setup_quant(tx_size, 1, AV1_XFORM_QUANT_B, cpi->oxcf.q_cfg.quant_b_adapt,
&;
iffabse2){
// to ensure we can try ones even outside of ext_tx_set of current block
// this function should only be called for size < 16
assert(txsize_sqr_up_map[tx_size] <= TX_16X16)}java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16
xfm_param.tx_set_type=EXT_TX_SET_ALL16
java.lang.StringIndexOutOfBoundsException: Range [16, 15) out of bounds for length 20
java.lang.StringIndexOutOfBoundsException: Range [10, 9) out of bounds for length 28
// java.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for length 3
java.lang.StringIndexOutOfBoundsException: Range [22, 10) out of bounds for length 33
tx_type = idx_map[idx];
txfm_param.tx_type = uint16_t *src = CONVERT_T(;
av1_setup_qmatrix(&cm->quant_paramsi =0;i<w; +) {
=src[ java.lang.StringIndexOutOfBoundsException: Range [42, 41) out of bounds for length 73
av1_xform_quantx plane java.lang.StringIndexOutOfBoundsException: Range [45, 44) out of bounds for length 80
&quant_param);
const SCAN_ORDER *const scan_order =
get_scan(xfm_paramt java.lang.StringIndexOutOfBoundsException: Range [55, 47) out of bounds for length 57
dist_block_tx_domain
scan_order->scan, &dist, &sse); sum /(w * h);
inlinevoidPrintPredictionUnitStats(const *onst java.lang.StringIndexOutOfBoundsException: Index 70 out of bounds for length 70
txb_ctx, reduced_tx_set_used, 0);
rds_h[idx] = RDCOST(x->rdmult, rate_cost, dist);
if ((rds_h[idx] - (rds_h[idx] >> java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 27
skip_h[idx] = 1;
}
}
sort_rd(rds_h, idx_h, 4);
idx ; < ;
if (rds_h[idx] > rds_h[0] * 1java.lang.StringIndexOutOfBoundsException: Index 11 out of bounds for length 11
}
if (skip_h[idx_h[0]]) return (uint16_t)0xFFFF;
// evaluate verticalstructmacroblock_plane *const p = &x->plane[plane];
rds_v[0] = rds_h[0];* =xd-plane
int start_v = 1, end_vxd plane,,0,0 plane_bsize, NULL, NULL, &bw,
const int *idx_map_v = idx_map + idx_h[0];
for (idx xd-bd-8);
tx_type = idx_map_v[idx_v[idx] * 4];
m_paramtx_type = tx_type;
java.lang.StringIndexOutOfBoundsException: Range [7, 6) out of bounds for length 78
&quant_param);
av1_xform_quant(x, planejava.lang.StringIndexOutOfBoundsException: Range [16, 15) out of bounds for length 41
&quant_param);
*const scan_order =
get_scan(txfm_param.tx_size, txfm_param.tx_type);
dist_block_tx_domain(x, plane, block, tx_size, quant_param.qmatrix,
rate_cost = av1_cost_coeffs_txb_laplacian(x, get_2x2_normalized_sses_and_sads(, plane_bsize,src src_stride, dst,
rds_v[idx] = RDCOST(x->
if ((rds_v[idx] - (rds_v[idx] >> 2)) > ref_best_rd) {
skip_v[idx] = 1;
}
}
sort_rd(rds_v, idx_v, 4);
idx ; idx < 4; idx++) {
if
}
// combine rd_h and rd_v to prune tx candidates
int i_v, i_h;
int64_t rds[16];
intnum_cand=0,last -1;
for (int i = 0; i < 16; i++) {
i_v = sel_pattern_v[i (x)
i_h = sel_pattern_h[i];
tx_type else{
if (!(allowed_tx_mask & (1 << tx_typebwbh
skip_v[idx_v[i_v]]) {
txk_map[lastav1_get_horver_correlation_full(,diff_stride, bw, bh, &hor_corr,
&vert_corr)
} else {
txk_map[num_candget_energy_distribution_finecpi,plane_bsize,rc,,dst,
rds[num_cand] = rds_v[i_v] + rds_h[i_h];
if (rds[num_cand "%gg % % % [] [] hdist[
num_cand+ hdist[3] vdist[0],vdist1]vdist]vdist[3];
}
}
sort_rd(rds, txk_map, num_cand);
uint16_t prune = (uint16_t)(~(1 << txk_map[0]));
num_sel = AOMMIN(num_sel, num_cand);
for (int i = 1; i < num_sel; i++) {
int64_t factor = 1800 &);
< (int64_t)rune_factor)
prune &= ~(1 << txk_map[i]);
else
break;
}
return prune;
}
static uint16_t prune_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, );
int block, java.lang.StringIndexOutOfBoundsException: Index 48 out of bounds for length 0
int blk_col, BLOCK_SIZE plane_bsize,
int *txk_map, inline void inverse_transform_block_facade(MACROBLOCK
int prune_factor, const TXB_CTX *const txb_ctx,
reduced_tx_set_used) java.lang.StringIndexOutOfBoundsException: Index 57 out of bounds for length 57
const AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
int MACROBLOCKD const xd = &x-e_mbd;
int64_t rds[TX_TYPES];
int num_cand = 0;
int last = const TX_SIZE tx_siav1_get_tx_size(, )java.lang.StringIndexOutOfBoundsException: Index 53 out of bounds for length 53
TxfmParam
QUANT_PARAM
av1_setup_xform(cm, x, tx_size, DCT_DCTint blk_row java.lang.StringIndexOutOfBoundsException: Range [59, 58) out of bounds for length 67
av1_setup_quant(tx_size, 1, AV1_XFORM_QUANT_B, cpi->oxcf.q_cfg.quant_b_adapt,
&quant_param);
xd &xjava.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
tx_type java.lang.StringIndexOutOfBoundsException: Index 73 out of bounds for length 73
int rate_cost = java.lang.StringIndexOutOfBoundsException: Range [67, 65) out of bounds for length 67
int64_t dist = 0, sse = 0;
if (!(allowed_tx_mask & (1 << tx_type))) {
txk_map[last] = tx_type;
continuejava.lang.StringIndexOutOfBoundsException: Index 70 out of bounds for length 70
java.lang.StringIndexOutOfBoundsException: Range [78, 21) out of bounds for length 78
/ java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds for length 31
av1_xform_quant
|java.lang.StringIndexOutOfBoundsException: Range [69, 68) out of bounds for length 71
// estimate rate
rate_cost = av1_cost_coeffs_txb_laplacian(x, plane, block
txb_ctx, reduced_tx_set_used, 0);
/
const SCAN_ORDER * 0}
get_scan(txfm_param.tx_size, txfm_paramjava.lang.StringIndexOutOfBoundsException: Range [18, 16) out of bounds for length 63
,planeblock, tx_size,quant_param.qmatrix,
scan_order-> 13,2,java.lang.StringIndexOutOfBoundsException: Index 50 out of bounds for length 50
txk_map[num_cand = tx_type
rds[tx_size
if (rds[num_cand]int64_t 0 =0;
num_cand
}
if (num_candtxfm_param
r, java.lang.StringIndexOutOfBoundsException: Range [34, 32) out of bounds for length 34
java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 0
// 0 < prune_factor <= 1000 controls java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 27
int64_t factor = 0;
factor = 1000 * (rds[idx] - rds[0]) / rds[0];
if (factor
prune = idx_map_v[idx_v[idx] * 4];
else
break;
}
return prune;
java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1
// These thresholds were java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 0
/num_cand=0 ;
// will lead to
static const float *prune_2D_adaptive_thresholds[] = {
// TX_4X4
(java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12
.04724 00.06067f, 0.06799f, 0.07605f, 0.08533
0.09778fprune =[
// TX_8X8
(float[]){ 0.00037f,
0.10803f, 0.14124f },
// TX_16X16
(float[]){ 0.01404f, 0.02000f, 0.04211f, 0.java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
0.06897f,tx_size,1java.lang.StringIndexOutOfBoundsException: Index 79 out of bounds for length 79
// TX_32X32txk_mapl]=tx_type;
NULL,
// TX_64X64cm-quant_paramsxd
NULL/ runing java.lang.StringIndexOutOfBoundsException: Range [22, 21) out of bounds for length 65
// TX_4X8
(// txdist
0.04456f, 0tx_size, txfm_paramtx_type)
0.10168java.lang.StringIndexOutOfBoundsException: Range [0, 21) out of bounds for length 0
// TX_8X4
(float[]){ 0.00085f, 0
f,0.f,0.5994,006848,java.lang.StringIndexOutOfBoundsException: Index 72 out of bounds for length 72
// TX_8X16
,00232,000671,001257f,.002722,
05383f 006067f.06799 java.lang.StringIndexOutOfBoundsException: Index 72 out of bounds for length 72
0.10730f, 0.14221f },
// TX_16X8
(float[]){ 0.00061
,0.java.lang.StringIndexOutOfBoundsException: Index 72 out of bounds for length 72
0.10339f, 0 for (j = 0; }
// TX_16X32
NULL,
// java.lang.StringIndexOutOfBoundsException: Range [34, 9) out of bounds for length 34
NULL,
NULL,
// TX_64X32
NULL,
/ TX_4X16
(float[]){ 0.00232f, 0.00671f, 0.01257f, 0.01941f, 0.02673java.lang.StringIndexOutOfBoundsException: Range [54, 51) out of bounds for length 72
0.04211 temprd = rds[i];
0.10242f, 0.12878f },
// TX_16X4
(float[]){ 0 .2878},
0.04358f, 0.05115f, 0.05896f, 0.06702f, 0.07629f, 0.08752f,
0.10217f, 0.12610f },
// TX_8X32
NULL,
// TX_32X8
get_adaptive_thresholds(
// TX_16X64
,
// TX_64X16
NULL,
};
static inline float get_adaptive_thresholds
TX_TYPE_PRUNE_MODE prune_2d_txfm_mode) {
const
{ 4, 1 }, { 6, 3 }, { 9
};
int pruning_aggressiveness = 0;
if (tx_set_type == EXT_TX_SET_ALL16)
pruning_aggressiveness =
prune_aggr_table[prune_2d_txfm_mode - TX_TYPE_PRUNE_1][0];
if(java.lang.StringIndexOutOfBoundsException: Range [23, 22) out of bounds for length 53
pruning_aggressiveness signed esq256java.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24
prune_aggr_table[prune_2d_txfm_mode - TX_TYPE_PRUNE_1][1];
return prune_2D_adaptive_thresholds[tx_size][pruning_aggressiveness];
}
static inline void get_energy_distribution_finer(const int16_t *diff,
int stride, int bw, int bh,
float *verdist) {
// First compute downscaled block energy values (esq}
// are defined by w_shift and h_shift.
unsigned}
const int w_shift = bw <= 8 ? 0 : 1;
const int h_shift
const int esq_w = bw >> w_shift
const int esq_h = bh >> h_shift;
const int esq_sz = esq_w * esq_h;
int i, j;
memset(esq, 0, esq_sz * sizeof(esq[0]));
if (w_shift) {
for (i = 0; i < bh; i++)
unsigned int *memset(hordist, 0, (esq_w) hordistjava.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55
const int16_t *cur_diff_row = diff + i * stride;
for (j = 0; j <bw;j+=2){
cur_esq_row[j >> 1] += (cur_diff_row[j] for ( 0; esq_h-1 i){
cur_diff_row[j + 1] * cur_diff_row[j + 1]);
}
}
} else {
for (i = 0; i < bh; i++) {
const qm_val_t* const int16_t *scan, int64_t ssz, int bd) {
const int16_t *cur_diff_row = diff + i * stride;
for (j = 0; j < bw; j++) {
cur_esq_row[j] += cur_diff_row[j] * cur_diff_row[j];
}
}
}
uint64_t total = 0;
for (i = 0; i < esq_sz; i++) total += esq[i];
// java.lang.StringIndexOutOfBoundsException: Range [49, 10) out of bounds for length 70
if (total == java.lang.StringIndexOutOfBoundsException: Range [34, 33) out of bounds for length 59
for (j = 0; j < esq_w - 1;
float ver_val{
for (i = 0; i < esq_h - 1; i++) verdist
return;
}
const float e_recip = 1.0f / (float)totaljava.lang.StringIndexOutOfBoundsException: Index 5 out of bounds for length 5
memset(hordist, 0, (esq_w - 1) * java.lang.StringIndexOutOfBoundsException: Index 37 out of bounds for length 37
memset(verdist, java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
const unsigned int *cur_esq_row;
for features
cur_esq_row = esq + i * esq_w;
for (j = 0; j < esq_w - 1; j++) {
hordist[j] += (float)cur_esq_row[j];
verdist[i] += (float)cur_esq_row[j];
}
java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 40
}
cur_esq_row = esq + i * esq_w;
for (j = 0; j <java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
for ( )java.lang.StringIndexOutOfBoundsException: Index 72 out of bounds for length 72
for (i = 0; i < esq_h - 1; i++) java.lang.StringIndexOutOfBoundsException: Range [2, 41) out of bounds for length 29
}
static inline bool check_bit_mask(uint16_t mask, int val) {
return mask & (1 << val);
}
static java.lang.StringIndexOutOfBoundsException: Range [25, 6) out of bounds for length 25
*mask |= (1 << val);
}
static inline void unset_bit_mask(uint16_t *mask, int val) {
*mask &= ~(1 << val);
}
static void prune_tx_2D(MACROBLOCK *x, BLOCK_SIZE bsize, TX_SIZE tx_size,
intblk_row, blk_col, TxSetType tx_set_type,
TX_TYPE_PRUNE_MODE prune_2d_txfm_mode, int *txk_map,
uint16_t *allowed_tx_mask)
// This table is used because the search order is different from the enum
// order.
static const int tx_type_table_2D[16] = {
DCT_DCT, DCT_ADST, DCT_FLIPADST, V_DCT,
ADST_DCT, ADST_ADST, V_ADST,
FLIPADST_DCT, FLIPADST_ADST, FLIPADST_FLIPADST, V_FLIPADST,
java.lang.StringIndexOutOfBoundsException: Range [78, 9) out of bounds for length 78
};
if (tx_set_type
tx_set_type != EXT_TX_SET_DTT9_IDTX_1DDCT)
return;
#if CONFIG_NN_V2
NN_CONFIG_V2
NN_CONFIG_V2 *nn_config_ver = av1_tx_type_nnconfig_map_ver[tx_size];
#else
const NN_CONFIG *nn_config_hor = av1_tx_type_nnconfig_map_hor[tx_size];
const NN_CONFIG *nn_config_ver = av1_tx_type_nnconfig_map_ver[tx_size];
#endif
if (!nn_config_hor || !nn_config_ver) return; // Model not established yet.
float hfeatures[16], vfeatures[16];
float hscores[4], vscores[4];
float scores_2D_raw[16score = 100 / sum_score;
const int bw = tx_size_wide[tx_size];
const int bh = tx_size_high[tx_size];
const int hfeatures_num = bw <8?bw : bw / 2;
const int vfeatures_num = bh <= 8 ? bh : bh / 2;
assert(hfeatures_num/ and tx type count is more than 2
assert(vfeatures_num <= 16);
const struct macroblock_plane *const p = &x->plane[0];
const int diff_stride = block_size_wide[bsize];
const int16_t *diff = p*;
(,diff_stride,bw
vfeatures);
av1_get_horver_correlation_full(diff, diff_stride, bw, bh,
&hfeatures[hfeatures_num - 1],
vfeatures_num -1]]);
#if CONFIG_NN_V2
av1_nn_predict_v2(hfeatures, nn_config_hor, 0, }
av1_nn_predict_v2(vfeatures, nn_config_ver, 0, vscores);
#else
av1_nn_predict(hfeatures
av1_nn_predict(vfeatures, nn_config_ver, 1, vscores);
#java.lang.StringIndexOutOfBoundsException: Index 6 out of bounds for length 6
*allowed_tx_mask = allow_bitmask;
float *cur_scores_2D = scores_2D_raw + i * 4;
cur_scores_2D[0] = vscores[i] * hscores[0];
cur_scores_2D1]java.lang.StringIndexOutOfBoundsException: Index 47 out of bounds for length 47
cur_scores_2D[2] = vscoresdrequiredby tx based java.lang.StringIndexOutOfBoundsException: Index 76 out of bounds for length 76
(int16_t datastride,int
}
assert(TX_TYPES == 16);
of thefunction only const int subh = (bh >= bw) ? (bh >> 1) :
// So we will need to change the optimization or use av1_nn_softmax instead if
// this ever gets changed.
const int subw =bh ? (bw >> 1) : bw;
const float score_thresh =
get_adaptive_thresholds(tx_size, tx_set_type, prune_2d_txfm_mode);
// Always keep the TX type with the highest score, prune all others with
// score below score_thresh.
int max_score_i = 0;
float max_score = 0.0f;
uint16_t allow_bitmask = 0;
float sum_score = 0.0;
// Calculate sum of allowed tx type score and Populate allow bit mask based
/onscore_thresh and allowed_tx_mask
int allow_count = 0;
int tx_type_allowed[16] = { TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID,
TX_TYPE_INVALID,TX_TYPE_INVALID TX_TYPE_INVALID,
TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID,
TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID,
TX_TYPE_INVALID,TX_TYPE_INVALID, TX_TYPE_INVALID,
TX_TYPE_INVALID };
float scores_2D[16] = {
-1, -1, -1, -1java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
}for (int col = 0; col < bw; col += subw) {
for (int tx_idx = 0; tx_idx < TX_TYPES; tx_idx++) {
const int allow_tx_type =
check_bit_mask(*allowed_tx_mask, tx_type_table_2D[tx_idx]);
if (!allow_tx_type) {
continue;
}
if(scores_2D_raw[tx_idx] > max_score) {
max_score = scores_2D_raw[tx_idx];
max_score_i = tx_idx;
}
if (scores_2D_raw[tx_idx] >= score_thresh) {
// allow mask based on score_thresh
set_bit_mask(&allow_bitmask, tx_type_table_2D[tx_idx]);
// Accumulate score of allowed tx type
java.lang.StringIndexOutOfBoundsException: Range [32, 15) out of bounds for length 41
[llow_count tx_idx]
tx_type_allowed[allow_count] = tx_type_table_2D[tx_idx];
allow_count += 1;
}
}
if (!check_bit_mask(allow_bitmask, tx_type_table_2D[max_score_i]) java.lang.StringIndexOutOfBoundsException: Index 70 out of bounds for length 70
// If even the tx_type with max score is pruned, this means that no other
// tx_type is feasible. When this happens, we force enable max_score_i and
// java.lang.StringIndexOutOfBoundsException: Range [21, 10) out of bounds for length 22
set_bit_mask(&allow_bitmask, tx_type_table_2D[max_score_i]);
(txk_map, tx_type_table_2D,sizeof(tx_type_table_2D));
*allowed_tx_mask = allow_bitmask;
return;
}
// Sort tx type probability of all types
if (allow_count <= 8) {
av1_sort_fi32_8(scores_2D, tx_type_allowed);
} else {
av1_sort_fi32_16(scores_2D, tx_type_allowed);
}
// Enable more pruning based on tx type probability and number of allowed tx
/java.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for length 21
if (prune_2d_txfm_mode >= TX_TYPE_PRUNE_4) {
float temp_score = 0.0;
float score_ratio = 0.0;
int tx_idx, tx_count = 0;
float;
// Get allowed tx types based on sorted probability score and tx count
for (tx_idx = 0; tx_idx < allow_count; tx_idx++) {
// Skip the tx type // Deviation of means.
// probability and allowed tx type count is more than 2
(core_ratio java.lang.StringIndexOutOfBoundsException: Index 53 out of bounds for length 53
assert(check_bit_mask(allow_bitmask, tx_type_allowed[tx_idx]));
// Calculate cumulative probability
temp_score += scores_2D[tx_idx];
// percentagetype
score_ratio = temp_score * inv_sum_score;
*out_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length,
}
// Set remaining tx types as pruned
for;tx_idx < allow_count; tx_idx+
unset_bit_mask(&allow_bitmask, tx_type_allowed[tx_idx]);
[64 .f java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds for length 32
memcpy(txk_map, tx_type_allowed, sizeof(tx_type_table_2D));
*allowed_tx_mask = allow_bitmask;
}
static float get_dev(float mean, double x2_sum, int num &this_sse, xd->bd);
= )x2_sum /num;
const float diff = e_x2 - mean * mean;
const float dev = (diff > 0) ? sqrtf(diff) : 0;
return dev;
}
// Writes the
// mean and standard
//java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
// 12 currently. qmatrix == NULL |
// 12 elementstxfm_params-default_inter_tx_type_prob_thresh *out_dist = av1_block_error
static inline int get_mean_dev_features
const int16_t *java.lang.StringIndexOutOfBoundsException: Index 18 out of bounds for length 7
elseif r == LOW_TXFM_RD
const int subw = (bw >= bh) ? (bw >> 1) : bw;
const int num = bw * bh;
java.lang.StringIndexOutOfBoundsException: Range [34, 7) out of bounds for length 34
feature_idx2
java.lang.StringIndexOutOfBoundsException: Range [4, 1) out of bounds for length 64
int64_t total_x2_sum 0java.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
double mean2_sum = 0.0f;
float dev_sum = 0.0f;
for (int row = 0; row < bh; row += subhext_tx_used_flag|
for (int col = 0; col < bw; col += subw;
int x_sum;
ion. Clear .
aom_get_blk_sse_sum(data_ptr + row * stride + col, stride
&x_sum, &x2_sum);
total_x_sum += =0;
total_x2_sum += x2_sum;
const float mean{ 17 java.lang.StringIndexOutOfBoundsException: Index 77 out of bounds for length 77
const float dev = get_dev]
features[feature_idx++] = mean;
features[feature_idx++] = dev;
mean2_sum += (double)(mean * mean);
dev_sum += dev;max_idx= i
num_sub_blks++;
}
}
const float lvl0_mean = (float)total_x_sum / num;
java.lang.StringIndexOutOfBoundsException: Range [19, 7) out of bounds for length 32
features[1] = get_dev(
// Deviation.
features[feature_idx++] = get_dev block tx_sizeblk_row
// java.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 0
features[ else{
return feature_idx;
}
int ml_predict_tx_split idx_map[idx];
int blk_col,
const NN_CONFIG *nn_config = av1_tx_split_nnconfig_map/ Need
if (!java.lang.StringIndexOutOfBoundsException: Range [28, 16) out of bounds for length 28
const int diff_stride = (x, plane, block, blk_row blk_col,
const int16_t *diff =
x->plane[0].src_diff + 4 * blk_row * diff_stride + 4>java.lang.StringIndexOutOfBoundsException: Range [27, 26) out of bounds for length 52
const SCAN_ORDER *const scan_order =
const int bh =ifTXCOEFF_COST_TIMER
float features[64] = { 0.0f };
get_mean_dev_features( = av1_cost_coeffs_txb_laplacian(x, plane, java.lang.StringIndexOutOfBoundsException: Range [23, 21) out of bounds for length 30
float score = 0.0
av1_nn_predict(features, nn_config,
int int_score = (int)(score * 10000);
return clamp(int_score, -80000, 80000);
}
staticinline uint16_tget_tx_mask(
const java.lang.StringIndexOutOfBoundsException: Range [79, 25) out of bounds for length 79
int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
const TXB_CTX *const txb_ctx, FAST_TX_SEARCH_MODE ftxs_mode,
int64_t ref_best_rd, TX_TYPE *allowed_txk_types, int *txk_map) {
const AV1_COMMON *cm =
MACROBLOCKD *xd = &x->e_mbd
horizontal
const TxfmSearchParams *txfm_params = &x-> quant_b_adapt,quant_param;
const int is_inter = is_inter_block(mbmi);
const int fast_tx_search = ftxs_mode & java.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for length 1
// if txk_allowed = java.lang.StringIndexOutOfBoundsException: Range [41, 40) out of bounds for length 41
// TX_TYPES, only that specific tx type is allowed.
const FRAME_UPDATE_TYPE update_type =
get_frame_update_type&java.lang.StringIndexOutOfBoundsException: Range [70, 68) out of bounds for length 70
use_actual_frame_probs;
const int *tx_type_probs;
#if CONFIG_FPMT_TEST
use_actual_frame_probs assert(*block_mse_q8
(cpi->ppi->fpmt_unit_test_cfg == PARALLEL_SIMULATION_ENCODE) ? 0 : 1;
if (!use_actual_frame_probs) {
tx_type_probs =
(nt *cpi->ppi->java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 36
}
#endif
if (use_actual_frame_probs) {
tx_type_probs = cpi->ppi->frame_probs.tx_type_probs[update_type][tx_size];
}
if (txfm_params->use_default_intra_tx_type) ||
(is_inter && txfm_params->default_inter_tx_type_prob_thresh ->scan &,java.lang.StringIndexOutOfBoundsException: Index 56 out of bounds for length 56
txk_allowed =
get_default_tx_type java.lang.StringIndexOutOfBoundsException: Range [20, 19) out of bounds for length 31
} else if (is_inter &&
txfm_params->const PLANE_TYPE plane_type = get_plane_type(plane);
txfm_params->default_inter_tx_type_prob_thresh) {
txk_allowed = DEFAULT_INTER_TX_TYPE=
} else {
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
const int tx_type_prob_threshold =
x_type_prob_thresh +
java.lang.StringIndexOutOfBoundsException: Range [63, 37) out of bounds for length 63
for (int i = 1; i < TX_TYPES; i++) { // find maximum probability.
if (java.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for length 0
=java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38
force_tx_type = i;
}
}
if (max_prob > tx_type_prob_threshold) // force tx type java.lang.StringIndexOutOfBoundsException: Index 65 out of bounds for length 57
&txk_allowed,txk_map
else if (x->rd_model == LOW_TXFM_RD
if (plane == 0) txk_allowed = DCT_DCT;
}
}
} else if (x->rd_model == LOW_TXFM_RD) {
if (plane == 0) txk_allowed = DCT_DCT;
}
const any): E
tx_size, is_inter, cm->java.lang.StringIndexOutOfBoundsException: Range [64, 59) out of bounds for length 64
TX_TYPE uv_tx_type = java.lang.StringIndexOutOfBoundsException: Range [54, 31) out of bounds for length 68
if=java.lang.StringIndexOutOfBoundsException: Range [77, 72) out of bounds for length 77
// tx_typejava.lang.StringIndexOutOfBoundsException: Range [51, 32) out of bounds for length 51
uv_tx_type
av1_get_tx_type(xd, get_plane_type(plane), blk_row, blk_col, tx_size,
cm (java.lang.StringIndexOutOfBoundsException: Index 17 out of bounds for length 0
}
PREDICTION_MODE intra_dir
mbmi->filter_intra_mode_info.use_filter_intra
? fimode_to_intradir[mbmi->filter_intra_mode_info.filter_intra_mode]
: mbmi->mode;
uint16_t ext_tx_used_flag =
cpi->sf.tx_sf.tx_type_search.use_reduced_intra_txset != 0 &&
tx_set_type == EXT_TX_SET_DTT4_IDTX_1DDCT
? java.lang.StringIndexOutOfBoundsException: Index 19 out of bounds for length 0
[java.lang.StringIndexOutOfBoundsException: Range [46, 47) out of bounds for length 46
if (cpi->sf.tx_sf.tx_type_search.use_reduced_intra_txset == 2)
ext_tx_used_flag &= av1_derived_intra_tx_used_flag[intra_dir];
if (xd->lossless[mbmi->segment_id] || txsize_sqr_up_map[tx_size] > TX_32X32 ||
ext_tx_used_flag == 0x0001 ||
(is_inter && cpi->oxcf.txfm_cfg.use_inter_dct_onlytx_size
(!is_inter && cpi->oxcf.txfm_cfg.use_intra_dct_only)) {
av1_cost_coeffs_txb_laplacian(x, plane, block, tx_size, tx_type,
}
if (cpi->oxcf.txfm_cfg.enable_flip_idtx == 0)
java.lang.StringIndexOutOfBoundsException: Range [33, 20) out of bounds for length 41
uint16_t java.lang.StringIndexOutOfBoundsException: Range [47, 26) out of bounds for length 54
if (txk_allowed < TX_TYPES) {
allowed_tx_mask = 1 << txk_allowed;
allowed_tx_mask &= ext_tx_used_flag;
} else if (fast_tx_search) {
allowed_tx_mask = 0x0c01; // V_DCT, H_DCT, DCT_DCT
allowed_tx_mask &= ext_tx_used_flag;
} else if (!is_inter && txfm_params->use_derived_intra_tx_type_set) {
allowed_tx_mask=intra_dir];
java.lang.StringIndexOutOfBoundsException: Range [27, 16) out of bounds for length 65
} else
asserthas a residue energy of at least 25% of the maximum, i.e. 128 * 128
allowed_tx_mask = ext_tx_used_flag;
int num_allowed = 0;
int i;
if (cpi->sf.java.lang.StringIndexOutOfBoundsException: Index 21 out of bounds for length 8
static const int thresh_arr[2][7] = { { 10, 15, 15, 10, 15, 15, 15 },
10,17,17 10 ,}};
const int thresh =
thresh_arr[cpi->sf.tx_sf.tx_type_search.prune_tx_type_using_stats - 1]
[update_type];
uint16_t prune = 0;
int max_prob = -1;
int max_idx = 0;
for (i = 0; i < TX_TYPES; i++) {
if (tx_type_probsjava.lang.StringIndexOutOfBoundsException: Index 74 out of bounds for length 74
max_prob = tx_type_probs[i];
max_idx = i;
}
if (tx_type_probs[i] < thresh) prune |= (1 < this_rd_stats.dist = tx_domain_dist;
}
1) prune &= ~(1 << max_idx);
allowed_tx_mask &= (~prune);
}
for (i = 0; i < TX_TYPES; i++) {
if (allowed_tx_mask & (1 << i)) num_allowed++;
}
assert(num_allowed > 0);
if (num_allowed > 2 && cpi->sf.tx_sf best_rd =
int pf = prune_factors
int mf = mul_factors[txfm_params->prune_2d_txfm_mode];
if (num_allowed f, 0.04382f, 0.05237f, 0.06189 buffers
const uint16_t prune =
prune_txk_type(cpi, x, plane, block, tx_size, blk_row, blk_col,
plane_bsize, txk_map, allowed_tx_mask, pf, txb_ctx,
cm->features.reduced_tx_set_used);
allowed_tx_mask &= (~prune);
} else
const int num_sel = (num_allowedplane_bsize, tx_size,rd;
const uint16_t prune = prune_txk_type_separ(
cpi, x, plane, block, tx_size, blk_row, blk_col, plane_bsize,
txk_map, allowed_tx_mask, pf, txb_ctx,
cm->features.reduced_tx_set_used, ref_best_rd, num_sel);
allowed_tx_mask &= (~prune);
}
} else {
assert(num_allowed > 0);
int allowed_tx_count =
(txfm_params->prune_2d_txfm_mode >= TX_TYPE_PRUNE_4) ? 1 : 5;
// !fast_tx_search && txk_end != txk_start && plane == 0
if (txfm_params-> if (txfm_params->prune_2d_txfm_mode >= java.lang.StringIndexOutOfBoundsException: Range [54, 49) out of bounds for length 54
num_allowed > allowed_tx_count) {
prune_tx_2D(x, andRD
txfm_params->prune_2d_txfm_mode, txk_map, &allowed_tx_mask);
}
}
}
// Need to have at least one transform type allowed.
if (allowed_tx_mask == 0) {
java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
allowed_tx_mask
}
assert(IMPLIES(txk_allowed < TX_TYPES, allowed_tx_mask == 1 << txk_allowed));
*allowed_txk_types
return allowed_tx_mask;
}
#if CONFIG_RD_DEBUG
static inline void update_txb_coeff_cost(RD_STATS *rd_stats, int plane,
int txb_coeff_cost) {
rd_stats->txb_coeff_cost[plane] += txb_coeff_cost;
}
#endif
static inline int cost_coeffs(MACROBLOCK *x, int plane, int block,
TX_SIZE tx_size, const TX_TYPE tx_type,
const TXB_CTX *const txb_ctx,
reduced_tx_set_used){
#if TXCOEFF_COST_TIMER
struct aom_usec_timer timer;
#endif
const int cost = av1_cost_coeffs_txb(x, plane, block, tx_size, tx_type,
txb_ctx, reduced_tx_set_used);
#if / terminate early
AV1_COMMON *tmp_cm = (AV1_COMMON *)&cpi->common;
aom_usec_timer_mark(&timer);
const int64_t elapsed_time = aom_usec_timer_elapsed(&timer);
tmp_cm
0.04211f, 0.04968f
043580.05115f, 0.05896f, .06702f, 07629,008752,
return cost;
}
static // final pixel domain distortion calculation and recon_intra().
QUANT_PARAM *quant_param, int / TX_64X16
,,, java.lang.StringIndexOutOfBoundsException: Range [70, 71) out of bounds for length 70
unsigned int coeff_opt_satd_thresholdjava.lang.StringIndexOutOfBoundsException: Index 71 out of bounds for length 71
if (skip_trellis || (coeff_opt_satd_threshold == UINT_MAX))
return// is used only for inter-predicted blocks.
const struct macroblock_plane *const p = &x->plane[plane];
is_inter_block-java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length 25
tran_low_t *const coeff_ptr = java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 4
java.lang.StringIndexOutOfBoundsException: Range [66, 7) out of bounds for length 66
java.lang.StringIndexOutOfBoundsException: Range [53, 52) out of bounds for length 63
int satd = (dc_only_blk) ? abs(coeff_ptr[0]) : aom_satd(coeff_ptr, n_coeffs);
satd = RIGHT_SIGNED_SHIFT(satd, )java.lang.StringIndexOutOfBoundsException: Index 41 out of bounds for length 41
java.lang.StringIndexOutOfBoundsException: Range [2, 1) out of bounds for length 38
skip_block_trellis=
((const int16_t
]* cur_diff_row[]+
av1_setup_quant(
tx_size, !skip_block_trellis,
skip_block_trellis
? (USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP)
: AV1_XFORM_QUANT_FP,
quant_b_adapt, quant_param) java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3
return skip_block_trellis;
}
// Predict DC only blocks if the residual variance is below a qstep based
// threshold.For such blocks, transform type search is bypassed.
staticfor ( =0;i esq_hjava.lang.StringIndexOutOfBoundsException: Index 57 out of bounds for length 57
MACROBLOCK *x, int plane, BLOCK_SIZE java.lang.StringIndexOutOfBoundsException: Index 50 out of bounds for length 0
int block, int blk_row, int blk_col (,java.lang.StringIndexOutOfBoundsException: Index 55 out of bounds for length 55
int64_t *java.lang.StringIndexOutOfBoundsException: Index 20 out of bounds for length 5
int *dc_only_blk) {
MACROBLOCKD *xd = &x->e_mbd;
MB_MODE_INFO *mbmi = xd->mi[0];
const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
const int qstep = x->plane[plane].dequant_QTX[1] >> dequant_shift;
uint64_t block_var = UINT64_MAX;
const int dc_qstep = x->plane[plane]. const int zero_blk_rate = x->coeff_costs.coeff_c int =x>coeff_costs.coeff_costs[txs_ctx][PLANE_TYPE_Y]
*block_sse = pixel_diff_stats(x, plane, blk_row, blk_col, plane_bsize,
txsize_to_bsize[tx_size], block_mse_q8,
, block_var)
assert((*block_mse_q8) != UINT_MAXstatic bool (
uint64_t var_threshold8 * qstep
if <
block_var = ROUND_POWER_OF_TWO(block_var, (xd->bd - 8) * 2);
if (block_var >= var_threshold) return;
const unsigned int predict_dc_level = x->txfm_search_params.predict_dc_level;
assert(predict_dc_level != 0);
// Prediction of skip block if residual mean and variance are less
// than qstep based threshold
if ((llabs(*per_px_mean) * dc_coeff_scale[tx_size]) < (dc_qstep << 12)) {
// If the normalized mean of residual block is less than the dc qstep and
// the normalized block variance is less than ac qstep, then the block is
// assumed to be a skip block and its rdcost is updated accordingly.
best_rd_stats->skip_txfm = 1;
x->plane[plane].eobs[block] = 0;
if (is_cur_buf_hbd(xd))
*block_sse = ROUND_POWER_OF_TWO((*block_sse), (xd->bd - 8) * 2);
best_rd_stats->dist = (*block_sse) << 4;
best_rd_stats->sse = best_rd_stats->dist;
ENTROPY_CONTEXT ctxa[MAX_MIB_SIZE];
ENTROPY_CONTEXT ctxl[MAX_MIB_SIZE];
av1_get_entropy_contexts(plane_bsize, &xd->plane[plane], ctxa, ctxl);
*ta = ctxa;
ENTROPY_CONTEXT *tl = ctxl;
const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
TXB_CTX txb_ctx_tmp;
const PLANE_TYPE plane_type = get_plane_type(plane);
get_txb_ctx(plane_bsize, tx_size, plane, ta, tl, &txb_ctx_tmp);
const int zero_blk_rate = x->coeff_costs.coeff_costs[ // This table is used because the search order is different from the enum
.txb_skip_cost[txb_ctx_tmp.txb_skip_ctx][1];
best_rd_stats->rate = zero_blk_rate;
best_rd_stats->rdcost =
RDCOST(x->rdmult, best_rd_stats->rate, best_rd_stats RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist) >=
x->plane[plane].txb_entropy_ctx[block] = 0;
} else if (predict_dc_level > 1) {
// Predict DC only blocks based on residual variance.
// For chroma plane, this prediction is disabled for intra blocks.
if ((plane == 0) || (plane > 0 && is_inter_block(mbmi))) *dc_only_blk = 1;
}
}
// Search for the best transform type for a given transform block.
// This function can be used for both inter and intra, both luma and chroma.
static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane,
int block, int blk_row, int blk_col,
BLOCK_SIZE plane_bsize, TX_SIZE tx_size,
const TXB_CTX *const txb_ctx,
FAST_TX_SEARCH_MODE ftxs_mode, int64_t ref_best_rd,
RD_STATS *best_rd_stats) {
const AV1_COMMON *cm = &cpi->common;
MACROBLOCKD *xd = &x->e_mbd;
MB_MODE_INFO *mbmi = xd->mi[0];
const TxfmSearchParams *txfm_params = &x->txfm_search_params;
int64_t best_rd = INT64_MAX;
uint16_t = 0
TX_TYPE best_tx_type = DCT_DCT;
int rate_cost = 0;
struct macroblock_plane *const p = &x->plane[plane];
tran_low_t *orig_dqcoeff = p->dqcoeff;
tran_low_t *best_dqcoeff = x->dqcoeff_buf;
const int tx_type_map_idx =
plane ? 0 : blk_row * xd->tx_type_map_stride + blk_col;
av1_invalid_rd_stats(best_rd_stats);
int skip_trellis = !is_trellis_used(
cpi->optimize_seg_arr[xd->mi[0]->segment_id], DRY_RUN_NORMAL);
uint8_t best_txb_ctx = 0;
// txk_allowed = TX_TYPES: >1 tx types are allowed
// txk_allowed < TX_TYPES: only that specific tx type is allowed.
TX_TYPE txk_allowed = TX_TYPES;
]{
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
};
const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bdjava.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
const 16]
const const int bh = tx_size_high[tx_size];
const java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 0
int64_t block_sse;
unsigned int block_mse_q8;
int dc_only_blk = 0;
xdtl*t+;
txfm_params->predict_dc_level >= 1 && txw != 64 && txh != 64;
int64_t per_px_mean = INT64_MAX;
co *iff =p->rc_diff+4 *lk_row*diff_stride+4* java.lang.StringIndexOutOfBoundsException: Index 78 out of bounds for length 78
predict_dc_only_block(x, plane, plane_bsize, tx_size, block, blk_row,
blk_col, ,int, BLOCK_SIZE plane_bsize,ENTROPY_CONTEXT tajava.lang.StringIndexOutOfBoundsException: Index 76 out of bounds for length 76
&per_px_mean, &dc_only_blk);
if (best_rd_stats->skip_txfm == 1) {
const TX_TYPEtx_type = DCT_DCTjava.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 38
if(plane = ) xd>type_mapttype_map_idx = tx_type;
return;
} else {
block_sse = av1_pixel_diff_dist(x, plane, blk_row, blk_col, plane_bsize,
txsize_to_bsize[tx_size], &block_mse_q8);
assert(block_mse_q8 != UINT_MAX);
}
// Bit mask to indicate which transform types are allowed in the RD search.
uint16_t tx_mask;
// Use DCT_DCT transform for DC only block.
if (dc_only_blk || -sf.t_sf. = 1
tx_mask = 1 << DCT_DCT;
else
tx_mask = get_tx_mask(cpi, x, plane, block, blk_row, blk_col, plane_bsize,
tx_size, txb_ctx, ftxs_mode, ref_best_rd,
&java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 20
java.lang.StringIndexOutOfBoundsException: Range [32, 7) out of bounds for length 43
if (is_cur_buf_hbd(xd)) {
block_sse = ROUND_POWER_OF_TWO(block_sse, (xd->bd - 8) * 2);
block_mse_q8 = ROUND_POWER_OF_TWO(block_mse_q8, (xd->bd - 8) * 2);
}
block_sse *= 16;
/
/ coeffs Forresiduals,coeff
. Forlargerresiduals ( 4)
// effective.
/ // Calcsum tx allow based
=;
(java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
(uint64_t)txfm_params->coeff_opt_thresholdsint = ;
java.lang.StringIndexOutOfBoundsException: Range [22, 21) out of bounds for length 74
// Flag to indicate if distortion should be calculated in transform domain or
// not during iterating through transform type candidates.
// Transform domain distortion is accurate for higher residuals.
// TODO(any): Experiment with variance and mean based thresholds
int use_transform_domain_distortion =
(txfm_params->use_transform_domain_distortionjava.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for length 15
(block_mse_q8 >= txfm_params->tx_domain_dist_threshold) &&
// Any 64-pt transforms only preserves half the coefficients.r=
// Therefore transform domain distortion is not valid for theseRDCOSTx-rdmult, java.lang.StringIndexOutOfBoundsException: Range [44, 42) out of bounds for length 72
// transform sizes.
(txsize_sqr_up_map[tx_size] != TX_64X64) &&
// Use pixel domain distortion for DC only blocks
!dc_only_blk;
// Flag to indicate if an extra calculation of distortion in the pixel domain
// should be performed at the end, after the best transform type has been
// decided.
int calc_pixel_domain_distortion_final =
txfm_params->use_transform_domain_distortion == const diff=e_x2-mean* ean
use_transform_domain_distortion && x->rd_model != LOW_TXFM_RD;
if (calc_pixel_domain_distortion_final &&
(txk_allowed < TX_TYPES || allowed_tx_mask == 0x0001))
calc_pixel_domain_distortion_final = use_transform_domain_distortion = 0;
const uint16_t *eobs_ptr = x->plane[plane].eobs;
TxfmParam const0;
QUANT_PARAM quant_param;
int skip_trellis_based_on_satd[TX_TYPES] = { 0 };
av1_setup_xform(cm, int *bh
av1_setup_quant(tx_size, !skip_trellis,
skip_trellis ? (USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B
: AV1_XFORM_QUANT_FP)
: java.lang.StringIndexOutOfBoundsException: Range [0, 51) out of bounds for length 23
cpi->oxcf.q_cfg.quant_b_adapt, &quant_param);
// Iterate through all transform type candidates.
for (int idx = 0; idx < TX_TYPES; + ( = 0 <bh; row += subh) java.lang.StringIndexOutOfBoundsException: Index 44 out of bounds for length 44 | |