| Quellcodebibliothek Statistik Leitseite products/Sources/formale Sprachen/C/LibreOffice/sw/qa/extras/rtfimport/data/ (LibreOffice Version 25.8.3.2©) Datei vom 5.10.2025 mit Größe 110 kB |
|
SSL tx_search.c
Interaktion und
|
|
|
/*int64_tround() / ) * Copyright (c) 2020, Alliance for Open Media. All rights reserved. * * This java.lang.StringIndexOutOfBoundsException: Index 10 out of bounds for lengt * the Alliance for Open Media Patent License 1.0. If the BSD const dst8 wjava.lang.StringIndex * was not distributed with this source java.lang.StringIndexOutOfBoundsException: Range .org/software If forjava.lang.StringIndexOutOfBoundsException: Index 74 out of bou Patent License 1.0 was not distributed with this source code in the */ #include <inttypes.h> #include "av1/common/cfl.h" # av1reconintra." java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 java.lang.StringIndexOutOfBoundsException: Range [13, 8) out of bounds for lengt #include "av1/common/idct.h" #include "av1/encoder/model_rd.h" #include d,tx_bsize,, #include "av1/encoder/rdopt_utils.h" #include "av1/encoder/sorting_network.h" #include "av1/encoder/tx_prune_model_weights.h" includeaencoderjava.lang.StringIndexOutOfBoundsException: Range [31, 25) out of #include "av1/encoder/txb_rdopt.h" #define PROB_THRESH_OFFSET_TX_TYPE 100 structrdcost_block_args { const AV1_COMPstatic inline void PrintPredictionUnitStats(const AV1_COMP *const cpi, MACROBLOCK * const *tile_data, ENTROPY_CONTEXT t_above[MAX_MIB_SIZE]; ENTROPY_CONTEXT t_left[MAX_MIB_SIZE]; RD_STATS java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for len int64_t current_rd; int64_t best_rd; intexit_earlystatic int64_tdist_block_px_domain( AV1_COMP , MACROBLOCK x, int incomplete_exit; FAST_TX_SEARCH_MODE ftxs_mode; int skip_trellis; }; RD_STATS rd_stats int64_t rd; int txb_entropy_ctx; TX_TYPE tx_type; } TxCandidateInfo; // origin_threshold * 128 / 100 static plane_bsize { { 64, 64, 64, 70, 60, 60, 68, 68, 68, 68, 68, }, { 88, 88, 88, 86, 87, 87, 68, 68, 68, 68, 68, 68, 68, 68, 68, 68, 88, 88, 86, 86, 68, 68, }, { 90, 93, 93, 90, 93, 93, 74, 74, 74, 74, 74, 74 74, }, }; // lookup table for predict_skip_txfm // int max_tx_size = max_txsize_rect_lookup[bsize]; // if (tx_size_high[max_tx_size] > 16 || tx_size_wide[max_tx_size] > 16) // max_tx_size = AOMMIN(max_txsize_lookup[bsize], TX_16X16); constmax_predict_sf_tx_size[BLOCK_SIZES_ALL = { TX_4X4, TX_4X8, const BLOCK_SIZE = txsize_to_bsize[; TX_16X16, TX_16X16, TX_16X16, TX_16X16, TX_16X16, java.lang.StringIndexOutOfBoundsExc TX_16X16,java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for l TX_8X8, TX_8X8, TX_16X16, TX_16X16, }; // look-up table for sqrt of number of pixels in a transform block // rounded up to the nearest integer. static const int sqrt_tx_pixels_2d[TX_SIZES_ALL) 12, 12, 23, 23, 32, 32,8, 8, 16, 16, 23, 23 }; // look-up table for number of top no-split RD Costs that should be considered // based on prune_inter_tx_split_rd_eval_lvl speed feature. static const int num_inter_tx_no_split_cand[2] = { 4, 3 }; static inline uint32_t get_block_residue_hash(MACROBLOCK *x, BLOCK_SIZE bsize) { int block_size_high[ const int cols = block_size_wide[bsize]; const int16_t *diff = x->plane[0].src_diff; const hash = av1_get_crc32c_value(&x->txfm_search_info.mb_rd_record->crc_calculator, *diff 2*rowsjava.lang.StringIndexOutOfBoundsException: Index 51 out of bounds for return (hash << 5) + bsize const int =lock_size_wide[[; } static bw, ; constint = blk_row * get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize(xd,plan const uint32_t hash java.lang.StringIndexOutOfBoundsException: Range [16, 15) out o = 1 if (ref_best_rd != INT64_MAX) { for const *dst =&d->p].dst.buf[dst_idx]const java.lang.StringIndexOutOfBoundsExceptio const (->d -const *qcoeff p>+BLOCK_OFFSETblock) // If there is a match in the mb_rd_record, fetch the RD decision and // terminate early. if (mb_rd_record->mb_rd_info[index].hash_value == hash) { const double dist_norm = (double)rd_stats->dist / num_samples; break; } } } java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 } static inline void fetch_mb_rd_infoassert( !NULL); RD_STATS *java.lang.StringIndexOutOfBoundsException: Range [52, 51) out of bounds MACROBLOCK *const java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds MACROBLOCKD *const xd fprintf( 16uint16_trecon16MAX_TX_SQUARE)java.lang.StringIndexO MB_MODE_INFO *const mbmi = xdconst int=p>stride mbmi->tx_size uint8_t*onst =p>.java.lang.StringIndexOutOfBoundsException: Index 40 o _tx_size, mb_rd_info-inter_tx_size)java.lang.StringIndexOutOfBoundsException: Ind av1_copy_array(xd->tx_type_map, mb_rd_info->tx_type_map, n4); *rd_stats = mb_rd_info->rd_stats; } int64_t av1_pixel_diff_dist(const MACROBLOCK *x, int plane, int blk_row, int blk_col, const BLOCK_SIZEconst *onst src_difff java.lang.StringIndexOutOfBoundsException: Range [0, 44) out of bounds for lengt int64_t sse = calculate_sse(xd, p pd,recon ONVERT_TO_BYTEPTR(recon16; int visible_rows, visible_cols; const MACROBLOCKD*xd =&x->e_mbd; get_txb_dimensions(xd, , plane_bsize, blk_row,blk_col ,NULL NULL, &visible_cols, &java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 0 const int diff_stride = block_size_wide[plane_bsize]; const int16_t *diff = x->plane[plane].src_diffcpi->>fn_ptr[plane_bsize]sdf(src,src_stri diff += ((blk_rowconst double sad_norm java.lang.StringIndexOutOfBoundsException: Ind java.lang.StringIndexOutOfBoundsException: Index 16 out of bounds for length 16 aom_sum_squares_2d_i16(diff, diff_stride, visible_cols, visible_rows); if (block_mse_q8 != NULL) { if (visible_cols > 0 && visible_rowsfprintf(fout, " %g %g", sse_norm, sad_norm); *block_mse_q8 = (unsigned int)((256 * java.lang.StringIndexOutOfBoundsException: Index 35 out of bo else *block_mse_q8 = UINT_MAX; } return sse; } // Computes the residual block's SSE and mean on all visible 4x4s in the // transform block static inline int64_t pixel_diff_stats( MACROBLOCK *x, int plane, int blk_row, int blk_col, const BLOCK_SIZE plane_bsizesse_norm_arr, sad_norm_arr); unsigned int *block_mse_q8, int64_t *per_px_mean, uint64_t *block_var) { int visible_rows, visible_cols; const MACROBLOCKD *xd = &x->e_mbd; get_txb_dimensions(xd, plane, plane_bsize, blk_row, blk_col, tx_bsize, NULL, NULL, visible_cols, &visible_rows); const int diff_stride = block_size_wide[plane_bsize]; int16_t * = x-[.; diff += recon =(uint8_t <;+[](< ; uint64_t sse = 0; int sum = 0; sse = aom_sum_sse_2d_i16(diff, diff_stride, visible_cols, visible_rows, &sum } if (visible_cols > 0 && visible_rows > 0) { ++{ int sign_sum = sum > 0 ? 1 : -1; *per_px_mean = (int64_t)(norm_factor * abs(sum)) << 7; *java.lang.StringIndexOutOfBoundsException: Range [45, 16) out of bounds for len *block_mse_q8 = (unsigned int)(norm_factor * (256 * sse(fout"%",java.lang.StringIndexOu *block_var = (uint64_t)(sse - (uint64_t)(fprintf(fout, " %d %d %d %d", q_step, x->rdmult, bw, } else { *block_mse_q8 = UINT_MAX; } return sse; } // Uses simple features on top of DCT coefficients to quickly predict // whether optimal RD decision is to skip encoding the residual. // The sse value is stored in dist. static int predict_skip_txfm(MACROBLOCK *x, BLOCK_SIZE bsize, int64_t *dist,model_rd_ss int reduced_tx_set) { *xfm_params =x>; const int bw = block_size_wide[bsize]; const int bh = block_size_high[bsize]; const MACROBLOCKD *xd = &x->e_mbd; const int16_t dc_q = (double)RDCOST(x->rdmult, model_rate, model_dist) / num_samples *ist = av1_pixel_diff_dist(x, 0, ,bsize bsize ; const int64_t mse = *dist / bw / bh; // Normalized quantizer takes the transform upscaling factor (8 for tx size // smaller than 32) into account. const int16_t normalized_dc_q = dc_q >> 3del_dist / num_samples; const int64_t mse_thresh = (int64_t)normalized_dc_q * normalized_dc_q / 8; // For faster early skip decision, use dist to compare against threshold so // that quality risk is less for the skip=1 decision. Otherwise, use mse // since the fwd_txfm coeff checks will take care of quality // TODO(any): Use dist to return 0 when skip_txfm_level is 1 int64_t java.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for len // Predict not to =get_highbd_diff_mean->b,p>stride,pd>.bufbuf, if (pred_err > mse_thresh) return 0; // Return as skip-dststride bw, bh; else if (txfm_params->skip_txfm_level >= 2) return 1; constmax_tx_size= [bsize; const int tx_h = tx_size_high[max_tx_size]; const int tx_w = tx_size_wide[max_tx_size]; DECLARE_ALIGNEDbw bh); TxfmParam param; param.tx_type = DCT_DCT; param.tx_size = max_tx_size; param.bd = xd->bd; param.is_hbd } param.lossless = 0; param.tx_set_type = av1_get_ext_tx_set_type( param.tx_size, is_inter_block(xd->mi[0]), java.lang.StringIndexOutOfBoundsException const java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for const uint32_t max_qcoef_thresh = skip_pred_threshold[bd_idx][bsize]; const int16_t *src_diff = x->plane[0].src_diff; const int n_coeff = tx_w * tx_h; const int16_t ac_q = av1_ac_quant_QTX(x->qindex, 0, xd->bd); const uint32_t * dc_q; const uint32_t ac_thresh = max_qcoef_thresh * ac_q; for (int row = 0; row < bh; row += tx_h) { for (int col = 0; col < bw; col += tx_w) { av1_fwd_txfm(src_diff + col, coefs, bw, ¶m); // Operatingfprintf(fout %%%" mean java.lang.StringIndexOutOfBoundsException: Range const uint32_t dc_coef = (((uint32_t)abs(coefs[0] if djava.lang.StringIndexOutOfBoundsException: Range [19, 17) out of bounds for l for (int i = 1; constuint32_t ac_coef = (((uint32_t)abs(coefs[i])) << 7); if (ac_coef >= ac_thresh) return 0; } } src_diff += tx_h * bw; } return 1; } //Used java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds fo static inline void set_skip_txfm(MACROBLOCK *x, RD_STATS *rd_stats, BLOCK_SIZE bsize, int64_t dist) { MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; const int n4 = bsize_to_num_blk(pi-.nter_sfinter_mode_rd_model_estimation=){ const TX_SIZE tx_size = max_txsize_rect_lookup[bsize]; assert(ile_data->inter_mode_rd_models[plane_bsize].ready); memset(mbmi->inter_tx_size, tx_size, sizeof(mbmi->inter_tx_size)); mbmi->tx_size = const int64_t overall_sse=get_sse(pi x); rd_stats->skip_txfm = 1; if (is_cur_buf_hbd(xd)) dist = ROUND_POWER_OF_TWO(dist, (xd->bd - 8) * 2); rd_stats->dist = rd_stats->sse = (dist << 4); // Though decision is to make the block as skip based on luma stats, // itis non . // intermediate non skip costs calculated by caller function will be // incorrect, if rate is set as zero (i.e., if zero_blk_rate is not // . Hence ispopulated tocodethe luma tx blks // as skip, the caller function based on final rd decision (i.e., skip vs // non-skip) sets the final rate accordingly. Here the rate populated // corresponds to coding all the tx blocks with zero_blk_rate (based on max tx // size possible) in the current block. Eg: For 128*128 block, rate would be // 4 * zero_blk_rate where zero_blk_rate corresponds constdoubleest_rdcost_norm = // block as 'all zeros' ENTROPY_CONTEXTctxa[MAX_MIB_SIZE]; ENTROPY_CONTEXT ctxl[MAX_MIB_SIZE]; av1_get_entropy_contexts(bsize, &xd->plane[0], ctxa, ctxl); ENTROPY_CONTEXT *ta = ctxa; ENTROPY_CONTEXT *tl = ctxl; const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); java.lang.StringIndexOutOfBoundsException: Range [17, 9) out of bounds for lengt get_txb_ctx(bsize, tx_size, 0, ta, tl, &txb_ctx est_rdcost_norm); const int zero_blk_rate = x->coeff_costs.coeff_costs[txs_ctx][PLANE_TYPE_Y] .txb_skip_cost[txb_ctx.txb_skip_ctx][1 rd_stats->rate = zero_blk_rate * (block_size_wide[bsize] >> tx_size_wide_log2[tx_size]) * (bsize]>>tx_size_high_log2[tx_size]); } static inline void save_mb_rd_info(int n4, uint32_t hash, #fclose(); MB_RD_RECORD *mb_rd_recordjava.lang.StringIndexOutOfBoundsException: Index 0 out int index; if (mb_rd_record->num < RD_RECORD_BUFFER_LEN) { java.lang.StringIndexOutOfBoundsException: Range [10, 9) out of bounds for lengt (mb_rd_record->index_start +int plane, int block, ++mb_rd_record->num; }{ index = mb_rd_record->index_start; mb_rd_record->index_start = (-common.features.educed_tx_set_used } MB_RD_INFO *const mb_rd_info = &mb_rd_record->mb_rd_info[index]; const MACROBLOCKD *const xd = &x->e_mbd; mb_rd_info (xd,dqcoeff, plane, tx_type, tx_size, recon, mb_rd_info->tx_size = mbmi->tx_size; av1_copy(mb_rd_info->inter_tx_size, mbmi->inter_tx_size); av1_copy_array(mb_rd_info->tx_type_map, xd->tx_type_map, n4); mb_rd_info->rd_stats = *rd_stats; } // Store the RD Cost of transform no-split !returnjava.lang.StringIndexOutOfBoundsExcept java.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for leng MACROBLOCK *x, const MB_MODE_INFO *return 16 * pixel_dist(cpi, x, plane, src, src_stride, re int MACROBLOCKDblk_row, blk_col, plane_bsize,tx_bsize); assert(blk_idx < MAX_TX_BLOCKS_IN_MAX_SB);tran_low_t *=pdqcoeff + BLOCK_OFFSET(block); (prune_inter_tx_split_rd_eval_lvljava.lang.StringIndexOutOfBoundsException: Inde if( = = return // Do not store forjava.lang.StringIndexOutOfBoundsException: Range [41, 7) out of b if (java.lang.StringIndexOutOfBoundsException: Range [42, 9) out of bounds for le asserts const mul_factors[5] {80,80 70,50, 30java.lang.StringIndexOutOfBoundsExceptio const int num_top_cand = num_inter_tx_no_split_cand[rune_inter_tx_split_rd_eval_lvl -1]; assert(num_top_cand <= TOP_INTER_TX_NO_SPLIT_COUNT); // Insertthe RDCost in sorted order for (int i = 0; i < num_top_cand; i++) { if tmp_rd< x>top_inter_tx_no_split_rdblk_idx][is inline void sort_rd(int64_t [],int [,int for (int j = num_top_cand - 1; j > i; j--) { x->top_inter_tx_no_split_rd[blk_idx][j] = x->top_inter_tx_no_split_rd[blk_idx][j - 1]; } x->top_inter_tx_no_split_rd[blk_idx][i] = tmp_rdfor ( break; stati recon_intra(const AV1_COMP *cpi, MACROBLOCK *x, int plane, } } // Prune the evaluation of transform split. static inline bool prune_tx_split_eval_using_no_split_rd (intblock,intblk_row, int blk const MACROBLOCK *x, const MB_MODE_INFO *mbmi, int64_t tmp_rd, int blk_idx, int prune_inter_tx_split_rd_eval_lvl) { (rune_inter_tx_split_rd_eval_lvl)return ; if (blk_idx == -1 || tmp_rd == INT64_MAX) return false; // Doint tempi; if mbmi-skip_mode != 0 || is_intrabc_block(mbmi)) return false; assert(prune_inter_tx_split_rd_eval_lvl <= 2java.lang.StringIndexOutOfBoundsExcepti temprd java.lang.StringIndexOutOfBoundsException: Range [53, 51) out of bounds fo =txki; assert(num_top_cand <= TOP_INTER_TX_NO_SPLIT_COUNT); // Do not prune if there is no valid top RD Cost for ( * common -blk_idx[num_top_cand1 = java.lang.StringIndexOutOfBoundsException: Index 74 out o returnfalse inter_tx_no_split_rdjava.lang.StringIndexOutOfBoundsException: Index 48 out of b return true; return false; } static int get_search_init_depth(int mi_width, int mi_height, intblk_row + java.lang.Str const SPEED_FEATURES *sf, size_search_method){ if (tx_size_search_method == USE_LARGESTALL) return MAX_VARTX_DEPTH; if (sf->tx_sf.tx_size_search_lgr_block) { if (if (do_quant) { mi_height > mi_size_high} return MAX_VARTX_DEPTH; } if (is_inter) { return (mi_height != mi_width) QUANT_PARAM quant_param_intra; : sf->tx_sf.inter_tx_size_search_init_depth_sqr (cm, java.lang.StringIndexOutOfBound } else { return) ? sf->tx_sf.intra_tx_size_search_init_depth_rect : sf->tx_sf tran_low_t* java.lang.StringIndexOutOfBoundsException: Range [35, 34) ou } } static inline void java.lang.StringIndexOutOfBoundsException: Range [52, 34) out of const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, int blk_col, java.lang.StringIndexOutOfB TX_SIZE tx_size, int depth, BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above, TXFM_CONTEXT *cpi->oxcf.q_cfg.quant_b_ada tats int64_t prev_level_rd, int64_t ref_best_rd, int *is_cost_valid, FAST_TX_SEARCH_MODE ftxs_mode,int blk_idx) // NOTE: CONFIG_COLLECT_RD_STATS has 3 possible values // 0: Do not collect any RD stats // 1: Collectjava.lang.StringIndexOutOfBoundsException: Range [22, 21) out of boun / units #if CONFIG_COLLECT_RD_STATS nline java.lang.StringIndexOutOfBoundsException: Range [2, 1) out of bounds for l const AV1_COMP *cpi, BLOCK_SIZE bsize, const uint8_t *src, int src_stride, const av1_optimize_b(cpi, ,plane,block, ,, , double *verdist) { const int bw = block_size_wide[bsize]; const int bh = block_size_high[bsize]; unsigned int esq[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; if (bsize < BLOCK_16X16 || (bsize >= BLOCK_4X16 && bsize <= BLOCK_32X8)) { // Special cases: calculate 'esq' values manually, as we don't have 'vf' // functions for the 16 (very small) sub-blocks of this block. const/ bd8 18bits(ncluding sign .); const int h_shift = (bh == 4) ? 0 : (//bd10 java.lang.StringIndexOutOfBoundsException: Inde assert(bw <= 32); (java.lang.StringIndexOutOfBoundsException: Range [14, 13) out of bounds for len assert((bw -1 >w_shift injava.lang.StringIndexOutOfBoundsException: Index 32 out of ifif(lane = &x-plane[plane]// below should fit in 54 bits, thus n java.lang.StringIndexOutOfBoundsExcep const best_tx_type != const uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst) error + dd( blk_row,,blk_col,tx_size,DC for (int i = 0; i < bh; ++i) for (int j = 0; j < bw; ++j) { const int index = (j >> w_shift) + ((i >> h_shift) << 2java.lang.StringIndexOutOfBoundsException: esq[index] += (src16[j + i * src_stridestatic unsigned pixel_dist_visible_only( (src16[j + i * src_stride] - dst16[j + i * dst_stride]); java.lang.StringIndexOutOfBoundsException: Index 9 out of bounds for length 9 } else{ for (int i = 0; i < bh; + const BLOCK_SIZE tx_bsize, int = sqcoeff s java.lang.StringIndexOutOfBo for (int j = const int index = (j >> w_shift) + ((i >> h_shift) << 2); esq[index] += (src[j + i * src_stride] - dst[ esq[index] += (src[j + i * src_stride] - dst[j (src[j + i * src_stride] - dst[j + i * dst_stride]); } } } java.lang.StringIndexOutOfBoundsException: Range [24, 8) out of bounds for leng const (bsize < BLOCK_SIZESifCONFIG_AV1_HIGHBITDEPTH assert TX_SIZE xd =x>; const BLOCK_SIZE subsize = (BLOCK_SIZE)f_index; assert(block_size_wide[bsize] == 4 * block_size_wide[subsize]); const qm_val_t *qmatrix cpi->ppi->fn_ptr[subsize].vf(src, src_stride, dst, dst_stride, &esq[0]); cpi>ppi->n_ptr[subsize]vf( + bw/4,src_stride java.lang.StringIndexOutOfBoundsExceptio dst_stride, &esq const int16_t *scan, return (unsigned int)ROUND_POWER_OF_TWO(sse64, (x cpi->ppi->fn_ptr[subsize].vf(src + bw / 2, src_stride, java.lang.StringIndexOutOfBoundsE , &sq2) cpi- )java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for lengt dst_stride, &esq[3]); src += bh / 4 * src_stride; dst+ bh *dst_stride; cpi->ppi->fn_ptr[subsize].vf(src, src_stride, dst, dst_stride, &esq[4]); cpi->ppi->fn_ptr / not dst_stride, &esq[5]); cpi->>[.( +bw /,src_stride, dst+ /2, dst_stride, &esq[6]); cpi->ppi->fn_ptr/the dst_stride, &esq[7]);. src += bh / 4 * src_stride; /4 *dst_stride; cpi->ppi->fn_ptr[subsize].vf(src, src_stride, dst, dst_strideint plane, const uint8_t *sr cpi->ppi->fn_ptr[subsize].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride, &esq[9]); cpi->ppi->fn_ptr[subsize].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride, &esq[10]); cpi->ppi->fn_ptr[subsize].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4, dst_stride, &esq[11]); src += bh / 4 * src_stride; dst += bh / 4 * dst_stride; cpi->ppi->fn_ptr[subsize].vf(src, src_stride, dst, dst_stride, &esq[12]); cpi->ppi->fn_ptr[subsize].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride, &esq[13]); cpi->ppi->fn_ptr[subsize].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride, &esq[14]); cpi->ppi->fn_ptr[subsize].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4, dst_stride, &esq[15]); } double total = (double)esq[0] + esq[1] + esq[2 const BLOCK_SIZE plane_bsize, esq[6] + esq[7] + esq[ { esq[12] + esq[13] + esq[14] + esq[15]; if (total > 0) { const const MACROBLOCKD *xd = &x->e_mbd; hordist[0] = ((java.lang.StringIndexOutOfBoundsException: Index 73 out of bounds f hordist[1] = ((double)esq[1] + esq[5] + esq[9] + esq[13]) * e_recip; hordist[2] = ((double)esq[2] + esq[6] + esq[10] + esq[14]) * e_recip; if (eed_4th) { hordist[3] = ((double)esq[3] + esq[7] + esq[11] + esq[15]) * e_recip; } java.lang.StringIndexOutOfBoundsException: Range [24, 11) out of bounds for leng verdist[1] = ((double)esq[4] + esq[5] + esq[6] + esq[7]) * e_recip; verdist[2] = (cpi ,src,src_stride,dst, if (need_4th) { verdist[3] = ((double)esq[12] + esq[13] + esq[14] + esq[15]) * e_recip; } } else { hordist[0] = verdist[0] = 0.25; hordist[1] = verdist[1] = 0.25; hordist[2] = verdist[2] = 0.25; if (need_4th) { txb_cols, visible_rows visible_cols); } } } static double get_sse_norm(java.lang.StringIndexOutOfBoundsException: Index 30 out static inline int64_t dist_block_px_domain(const AV1_COMP *cpi, MACROBLOCK *x, for int ;j < ;++){ for (int i = 0; i < w; ++i) { const =diff[* stride+i; sum += err * err; } } assert(w > 0 && h > 0); returnsum w*h; } static double get_sad_norm(const int16_t *diff, int stride, int w, int h) { double sum = 0.0; for (const =txsize_to_bsize[] for (int i = 0; i < w; ++i) { sum += abs(diff[j * stride + i]); } } assert(w > 0 && h const src_stride =x-plane[lane.src.stride; return sum /(w *h); } aticinline void get_2x2_normalized_sses_and_sads( const AV1_COMP * cpi BLOCK_SIZE tx_bsize,const uint8_t *const src, int src_stride, const uint8_t *const dst, int dst_stride, const int16_t *const src_diff, int const int src_idx = (blk_row * src_stride + blk_col) << MI_SIZ double *const sad_norm_arr) { const = get_partition_subsize(tx_bsize, PARTITION_SPLIT); if (if (==BLOCK_INVALID / calculatestats const int half_width = block_size_wide[tx_bsize] / 2; const int half_height = block_size_high[tx_bsize] / 2; for =; row <;++row { for (int col = 0; col < 2; ++col) { const int16_t *const this_src_diff = src_diff + row * half_heightassert(! NULL)java.lang.StringIndexOutOfBoundsException: if (sse_norm_arr) { sse_norm_arr[row * 2 + col] = get_sse_norm(this_src_diff, diff_stride, half_width, half_height); } if (sad_norm_arr) { sad_norm_arr[row * 2 + col] = get_sad_norm(this_src_diff, diff_stride, half_width, half_height); } } } } else { // use function pointers to calculate stats const int half_width = block_size_wide[tx_bsize_half]; const int half_height = block_size_high[tx_bsize_half]; const int num_samples_half = half_width * half_height; for (int row = 0; row < 2; ++row) { for (int col = 0; col < 2; ++col) { constuint8_t *constthis_src = src + row * half_height * src_stride + col * half_width; const uint8_t *const this_dst = dst onvolve_copy((dst) dst_stride, CONVERT_TO_SHORTPTR(recon), MAX_TX_SIZE, bsw, bsh); unsigned int this_sse; cpi->ppi->fn_ptr[tx_bsize_half].vf(this_src, src_stride else { dst_stride, &this_sse); sse_norm_arr[row * 2 + col] = (double)this_sse / num_samples_half; } if (sad_norm_arr) { const unsigned int this_sad = cpi->ppi->fn_ptr[tx_bsize_half].sdf( this_src, src_stride, this_dst, dst_stride } sad_norm_arr[row * 2 + col] = (double#lse } } } } } #if CONFIG_COLLECT_RD_STATS == 1 static aom_convolve_copy(st,dst_stride, java.lang.StringIndexOutOfBoundsException: double sum = 0java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for constPLANE_TYPE plane_type = get_plane_type(plane); for (int i = 0; i < w; ++i) { sum += diff[j * TX_TYPE tx_type = av1_get_tx_type(xd, plane_type, blk_row, blk_col, tx_size } } assert(w > 0 && h > 0); return sum / (w * h); } static void PrintTransformUnitStats const AV1_COMP *const cpi, MACROBLOCK *x, const RD_STATS *const rd_stats, int blk_rowcpi>common.features cpi>ommoneatures.); TX_TYPE tx_type, int64_t rd) { if (rd_stats->rate == INT_MAX || rd_stats->dist == INT64_MAX) java.lang.StringIndexOutOfBo // Generate small 16 pixel_dist(pi ,,, tatic unsigned int seed = 21743; if (lcg_rand16(&seed) % 256 > 0) return; const char output_file[] = "tu_stats.txt"; FILE *fout = fopen(output_file, "a"); if (!fout) return; const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size]; const MACROBLOCKD *const xd = &x->e_mbd; java.lang.StringIndexOutOfBoundsException: Range [11, 7) out of bounds for lengt java.lang.StringIndexOutOfBoundsException: Range [41, 8) out of bounds for lengt const struct macroblockd_plane *const pd = &xd->plane[plane]; const int txw = tx_size_wide[tx_size]; const int txh = tx_size_high[tx_size]; const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const int q_step = p->dequant_QTX[1] >> dequant_shift; const int num_samples = txw * txh; const rate_norm=()rd_statsrate/ ; const double dist_norm = (double)rd_stats->dist / num_samples; fprintf(fout, "%g %g", rate_norm, dist_norm); const int src_stride = p>rc.; const uint8_t *const src = &p->src.buf[(blk_row * src_stride + blk_col) << MI_SIZE_LOG2]; const int dst_stride = pd->dst.stride; const uint8_t *const dst = &pd->dst.buf[(blk_row * java.lang.StringIndexOutOfBoundsException: Index 34 out of b unsigned int if (qmatrix == NULL || !x->txfm_search_params.use_qm_dist_metric) { cpi->ppi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse); const double sse_norm = (double)sse / num_samples; const unsigned int sad = cpi->ppi->fn_ptr[tx_bsize].sdf(src, src_stride, dst, dst_stride); const double sad_norm = (double)sad / num_samples; fprintf(fout, " %g %g", sse_norm, sad_norm); const int diff_stride = block_size_wide[plane_bsize]; *out_dist RIGHT_SIGNED_SHIFT(*out_dist, shift); &p->src_diff[(blk_row * diff_stride + blk_col) << MI_SIZE_LOG2]; double sse_norm_arr[4], sad_norm_arr[4]; get_2x2_normalized_sses_and_sads(cpi, tx_bsize, src, src_stride, dst, dst_stride, src_diff, diff_stride, sse_norm_arr, sad_norm_arr); for (int i = 0; i < 4; ++i) { fprintf(fout, " %g", sse_norm_arr[i]); } for (int i = 0; i < 4; ++i) { fprintf,"%", sad_norm_arr[i]); } const TX_TYPE_1D tx_type_1d_row = htx_tab[tx_type]; const TX_TYPE_1D tx_type_1d_col = vtx_tabint blk_row, int blk_col, BLOCK_SIZE plane_bsize fprintf(fout, " %d %d %d ,int prune_factor TXB_CTX const txb_ctx, tx_size_high[tx_size], tx_type_1d_row, tx_type_1d_col); int model_rate; int64_t model_dist; model_rd_sse_fn[const AV1_COMMON *cm = &pi-common; &model_rate, &model_dist); const double model_rate_norm = (double)model_rate / num_samples; java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 fprintf(fout, intidx; const double mean = get_mean(src_diff, diff_stride, txw, txh); float hor_corr, vert_corr; av1_get_horver_correlation_full(src_diff, diff_stride, txw, txh, &hor_corr, &vert_corr); fprintf(fout, " %g %g %g", mean, hor_corr, vert_corr); double hdist[4] = { 0 }, vdist[4] = { 0 }; distribution_fine(cpi, tx_bsize, src, src_stride, dst, dst_stride, 1, hdist, vdist); fprintf(fout, " %g %g %g %g %g %g %g %g", hdist[0], hdist[1], hdist[2], hdist[3], vdist[0], vdist[1], vdist[2], vdist[3])intskip_v[] = 0}java.lang.StringIndex fprintf(fout, " %d %" PRId64, x->rdmult, rd); out, "\n"); fclose(fout); } #endif // CONFIG_COLLECT_RD_STATS == 1 #if CONFIG_COLLECT_RD_STATS >= 2 static int64_t get_sse(const AV1_COMP *cpi, const MACROBLOCK *x) { const AV1_COMMON *cm = &cpi->common; const int num_planes = av1_num_planes(cm); const MACROBLOCKD *xd = &x->e_mbd; onstMB_MODE_INFO *mbmi = xd->mi[0]; int64_t total_sse = 0; for (int plane = 0; plane < num_planes; ++planeHDCT, H_ADST, H_FLIPADST, IDTX const struct macroblock_plane *const p = &x->plane[plane]; const struct macroblockd_plane *const pd = &xd->plane[plane]; const BLOCK_SIZE bs = const int sel_pattern_v[16] = { unsigned int sse; if (plane) continue; cpi->ppi->fn_ptr[bs].vf(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, &sse); total_sse += sse; } total_sse <<= 4; return total_sse; } static int get_est_rate_dist(const TileDataEnc *tile_data, BLOCK_SIZE bsize, int64_t sse est_residue_cost, int64_t *est_dist) { const InterModeRdModel * }java.lang.StringIndexOutOfBoundsException: Index 4 out of if (md->ready) { QUAN quant_param; *est_residue_cost = 0; *est_dist = sse; TxfmParam txfm_param; } else { *est_dist = (int64_t)round(md->dist_mean); const java.lang.StringIndexOutOfBoundsException: Range [48, 49) out of bounds for // Clamp estimated rate cost by INT_MAX / 2. // TODO(angiebird@google.com): find better quant_param)java.lang.StringIndexOutOfBou if ((est_ld) < 1-2){ *est_residue_cost = INT_MAX / 2; } else { double est_residue_cost_dbl = ((sse - md->dist_mean) / est_ld); if (est_residue_cost_dbl < 0) { *est_residue_cost = 0; else { *est_residue_cost = (int)AOMMIN((int64_t)round(est_residue_cost_dbl) tx_set_type EXT_TX_SET_ALL16; } } if (*est_residue_cost <= int rate_cost = 0; *est_residue_cost = 0; *est_dist = sse; }int64_t dist = 0, sse = 0; } return 1; } return 0; } static double get_highbd_diff_mean(const uint8_t *src8, int src_stride, for (idx = 0; idx < 4; const uint8_t *dst8, int dst_stride, int w, int h) { constO_SHORTPTRsrc8)java.lang.StringIndexOutOfBoundsException: Index 50 out of b const uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); double sum = 0.0; for (int j = 0; j < h; ++j) { for (nti= 0; <w;+i)java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds const int diff [ *src_stride + i] - dst[j * dst_stride + i]; sum += diff; } } assert(w > 0 && h > 0); return sum / (w av1_xform_quant(x, plane, block,blk_row, blk_col, plane_bsize, &txfm_param, } static double get_diff_mean(const uint8_t *src, int src_stride, const uint8_t *dst, int dst_stride, int w, int h) { double sum = 0.0; for (int j = 0; j < h; ++j) { for ( get_scant.x_size, txfm_param.tx_type); const int diff = src[j * src_stride + i] - dst[j * dst_stride + i]; sum += diff; } } assert(w > 0 && h > 0); eturn/ } staticinline (const AV1_COMP*onst cpi, const TileDataEnc *tile_data, MACROBLOCK *x, const RD_STATS *const rd_stats, BLOCK_SIZE if (rd_stats->rate == INT_MAX || rd_stats->dist == INT64_MAX) return; if (cpi->sf.inter_sf.inter_mode_rd_model_estimation == 1 && (tile_data == NULL || !tile_data->inter_mode_rd_models[plane_bsize].ready)) return; (void)tile_data; // Generate small sample to restrict output size. static unsigned int seed = 95014; if ((lcg_rand16(&seed) % (1for(=1 idx < 4 idx++) { 1) return; const char output_file[] = "pu_stats.txt"; FILE *fout = fopen(output_file, "a"); if (!fout) return; MACROBLOCKD *const xd = &x->e_mbd; const int plane = 0; struct java.lang.StringIndexOutOfBoundsException: Range [26, 25) out of bounds fo struct macroblockd_plane pd=&xd>plane[plane]; const int diff_stride = block_size_wide[plane_bsize]; int bw, bh; get_txb_dimensions(xd, , plane_bsize,0 0, &bh); const int num_samples = bw * bh; const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const int q_step = p->dequant_QTX[1] >> dequant_shift; const int shift = (xdt shift=(xd>bd 8; const double rate_norm = (double)rd_stats->rate / num_samples; txfm_param.java.lang.StringIndexOutOfBoundsException: Range [23, 22) out of boun const double rdcost_norm = (double)RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist) / num_samples; fprintf(fout, "%g %g %g", rate_norm, dist_norm, rdcost_norm); const int src_stride = p->src.stride; const uint8_t *const src = p->src.buf; const int dst_stride = pd->dst.stride; const uint8_t *const dst = pd->dst.buf; const int16_t *const src_diff = p->src_diff; int64_t sse = calculate_sse(xd, p, pd, bw, bh); const double sse_norm = (double)sse / num_samples; const unsigned int sad = cpiconst SCAN_ORDER *const java.lang.StringIndexOutOfBoundsException: Index 40 out const double sad_norm = (double)sad / (1 << num_pels_log2_lookup[plane_bsize]); fprintf(fout, " %g %g", sse_norm, sad_norm); double sse_norm_arr[4], sad_norm_arr[4]; cpi,plane_bsize,,src_stride dst_stride, src_diff, diff_stride, txb_ctx, reduced_tx_set_used, 0); if (shift) { for (int k = 0; k < 4; ++k) sse_norm_arr[k] /= (1 << (2 * shift)); for (int k = 0; k < 4; ++k) sad_norm_arr[k] /= (1 << shift); } for (int i = 0; i < 4; ++i) { fprintf(fout, " %g", sse_norm_arr[i]); } for (int i = 0; i < 4; ++i) { fprintf(fout, " %g", sad_norm_arr[i]); } fprintf(fout, " %d %d %d %d", q_step, x->rdmult, bw, bh for ( =1java.lang.StringIndexOutOfBou int model_rate; int64_t model_dist; model_rd_sse_fn[MODELRD_CURVFIT](cpi, x, plane_bsize, plane, sse, num_samples, &model_rate, &model_dist); const double model_rdcost_norm = (double)RDCOST(x->rdmult, model_rateint =0 last =TX_TYPES- 1; const double model_rate_norm = (double)model_rate / num_samples; const double model_dist_norm = (double)model_dist / num_samples; fprintf(fout, " %g %g %g", java.lang.StringIndexOutOfBoundsException: Index 39 out of model_rdcost_norm); double mean; ifis_cur_buf_hbd(d){ mean = get_highbd_diff_mean(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh); } else { mean = get_diff_mean(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh); } mean /= (1 << shift); float hor_corr, vert_corr; av1_get_horver_correlation_fullsrc_diff, , ) fprintf(fout, " %g %g %g", mean, hor_corr, vert_corr); double hdist[4] = { 0 }, vdist[4] = { 0 }; (cpi , rc src_stride java.lang.StringIndexOutOfBoundsException: Index 70 out of boun dst_stride, 1, hdist, vdist); fprintf(fout, " g %g %g %g %g% g",hdist0,hdist[], 2], ,vdist [] [2, 3]; if (cpi->sf.inter_sf.inter_mode_rd_model_estimation == 1) { assert(tile_data->inter_mode_rd_models[java.lang.StringIndexOutOfBoundsException: const int64_t overall_sse = java.lang.StringIndexOutOfBoundsException: Range [0, 39) int est_residue_cost = 0; int64_t est_dist = 0; get_est_rate_dist(tile_data, plane_bsize, overall_sse, &est_residue_cost, est_dist; const if (factor pjava.lang.StringIndexOutOfBoundsException: Range [39, 38) out of const double est_dist_norm = (java.lang.StringIndexOutOfBoundsException: Range [4, 1 const double est_rdcost_norm = (double)RDCOST(x->rdmult, java.lang.StringIndexOutOfBoundsException: Index 48 out fprintf(fout, " %g %g %g",java.lang.StringIndexOutOfBoundsException: Index 0 out of est_rdcost_norm) } fprintf(fout, "\n"); fclose(fout); } #endif // CONFIG_COLLECT_RD_STATS >= 2 #endif // CONFIG_COLLECT_RD_STATS static *const x, int plane, int block, int blk_row, int blk_col, int eobint ){ if (!eob) return; struct macroblock_plane *const p = &x->plane[plane]; * &x>java.lang.StringIndexOutOfBoundsException: Index 36 out of bounds for length tran_low_t *dqcoeff = p->dqcoeff + const PLANE_TYPE plane_type = java.lang.StringIndexOutOfBoundsException: Index 43 ou ze = plane xd; const TX_TYPE tx_type = av1_get_tx_type(xd, plane_type, blk_row, blk_col, tx_size, reduced_tx_set); struct macroblockd_plane *const pd = &xd->plane[plane]; const int dst_stride = pd->dst.stride; uint8_t av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size TxfmParam txfm_param; dst_stride, eob, reduced_tx_set); } static inline void recon_intra(const AV1_COMP *cpi, MACROBLOCK *x, int plane, block,int blk_row,int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const TXB_CTX *const txb_ctx, int skip_trellis, TX_TYPE best_tx_type, int do_quant, int *rate_cost, uint16_t best_eob) { const AV1_COMMON *cm = &cpi->common; MACROBLOCKD * = &->e_mbd; MB_MODE_INFO *mbmi = xd->mi[0]; const int is_inter = is_inter_block(mbmi); if (!is_inter && best_eob && (blk_row + tx_size_high_unit[tx_size] < mi_size_high[plane_bsize] || blk_col + tx_size_wide_unit[tx_size] < mi_size_wide[plane_bsize])) { // if the quantized coefficients are stored in the dqcoeff buffer, we don't // need to do transform and quantization again. if (do_quant) { TxfmParam txfm_param_intra; QUANT_PARAM quant_param_intra; av1_setup_xform(cm, x, tx_size, best_tx_type, &txfm_param_intra); av1_setup_quant(tx_size, !skip_trellis, skip_trellis ? (USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP) : AV1_XFORM_QUANT_FP, cpi->oxcf.q_cfg.quant_b_adapt, &quant_param_intra); av1_setup_qmatrix(&cm->quant_params, xd, plane, tx_size, best_tx_type, &quant_param_intra); av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param_intra, &quant_param_intra); if (quant_param_intra.use_optimize_b) { av1_optimize_b(cpi, x, plane, block, tx_size, best_tx_type, txb_ctx, rate_cost); } } inverse_transform_block_facade(x, plane, block, blk_row, blk_col, x->plane[plane].eobs[block], cm->features.reduced_tx_set_used); // This may happen because of hash collision. The eob stored in the hash // table is non-zero, but the real eob is zero. We need to make sure tx_type // is DCT_DCT in this case. if (plane == 0 && x->plane[plane].eobs[block] == 0 && best_tx_type != DCT_DCT) { update_txk_array(xd, blk_row, blk_col, tx_size, DCT_DCT); } } } static unsigned pixel_dist_visible_only( const AV1_COMP *const cpi, const MACROBLOCK *x, constuint8_t *src, const int src_stride, const uint8_t *dst, const int dst_stride, const BLOCK_SIZE tx_bsize, int txb_rows, int txb_cols, int visible_rows, int visible_cols) { unsigned sse; if (txb_rows == visible_rows && txb_cols == visible_cols) { cpi->ppi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse); return sse; } #if CONFIG_AV1_HIGHBITDEPTH const MACROBLOCKD *xd = &x->e_mbd; if (is_cur_buf_hbd(xd)) { uint64_t sse64 = aom_highbd_sse_odd_size(src, src_stride, dst, dst_stride, visible_cols, visible_rows); return (unsigned int)ROUND_POWER_OF_TWO(sse64, (xd->bd - 8) * 2); } #else (void)x; #endif sse = aom_sse_odd_size(src, src_stride, dst, dst_stride, visible_cols, visible_rows); return sse; } // Compute the pixel domain distortion from src and dst on all visible 4x4s in // the // transform block. static unsigned pixel_dist(const AV1_COMP *const cpi, const MACROBLOCK *x, int plane, const uint8_t *src, const int src_stride, const uint8_t *dst, const int dst_stride, int blk_row, int blk_col, const BLOCK_SIZE plane_bsize, const BLOCK_SIZE tx_bsize) { int txb_rows, txb_cols, visible_rows, visible_cols; const MACROBLOCKD *xd = &x->e_mbd; get_txb_dimensions(xd, plane, plane_bsize, blk_row, blk_col, tx_bsize, &txb_cols, &txb_rows, &visible_cols, &visible_rows); assert(visible_rows > 0); assert(visible_cols > 0); unsigned sse = pixel_dist_visible_only(cpi, x, src, src_stride, dst, dst_stride, tx_bsize, txb_rows, txb_cols, visible_rows, visible_cols); return sse; } static inline int64_t dist_block_px_domain(const AV1_COMP *cpi, MACROBLOCK *x, int plane, BLOCK_SIZE plane_bsize, int block, int blk_row, int blk_col, TX_SIZE tx_size) { MACROBLOCKD *const xd = &x->e_mbd; const struct macroblock_plane *const p = &x->plane[plane]; const uint16_t eob = p->eobs[block]; const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size]; const int bsw = block_size_wide[tx_bsize]; const int bsh = block_size_high[tx_bsize]; const int src_stride = x->plane[plane].src.stride; const int dst_stride = xd->plane[plane].dst.stride; // Scale the transform block index to pixel unit. const int src_idx = (blk_row * src_stride + blk_col) << MI_SIZE_LOG2; const int dst_idx = (blk_row * dst_stride + blk_col) << MI_SIZE_LOG2; } const uint8_t *dst = &xd->plane[plane].dst.buf[dst_idx]; const tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block); assert(cpi != NULL); assert(tx_size_wide_log2[0] == tx_size_high_log2[0]); uint8_t *recon; DECLARE_ALIGNED(16, uint16_t, recon16[MAX_TX_SQUARE]); #if CONFIG_AV1_HIGHBITDEPTH if (is_cur_buf_hbd(xd)) { recon = CONVERT_TO_BYTEPTR(recon16); aom_highbd_convolve_copy(CONVERT_TO_SHORTPTR(dst), dst_stride, CONVERT_TO_SHORTPTR(recon), MAX_TX_SIZE txfm_param.tx_type = tx_type; } else { recon = (uint8_t *)recon16; aom_convolve_copy(dst, dst_stride, recon, MAX_TX_SIZE, bsw, bsh); } #else recon = (uint8_t *)recon16; aom_convolve_copy(dst, dst_stride, recon, MAX_TX_SIZE, bsw, bsh); #endif const PLANE_TYPE plane_type = get_plane_type(plane); TX_TYPE tx_type = av1_get_tx_type(xd, plane_type, blk_row, blk_col, tx_size, cpi->common.features.reduced_tx_set_used); av1_inverse_transform_block(xd, dqcoeff, plane, tx_type, tx_size, recon, MAX_TX_SIZE, eob, cpi->common.features.reduced_tx_set_used); return 16 * pixel_dist(cpi, x, plane, src, src_stride, recon, MAX_TX_SIZE, blk_row, blk_col, plane_bsize, tx_bsize); } // pruning thresholds for prune_txk_type and prune_txk_type_separ static const int prune_factors[5] = { 200, 200, 120, 80, 40 }; // scale 1000 static const int mul_factors[5] = { 80, 80, 70, 50, 30 }; // scale 100 // R-D costs are sorted in ascending order. static inline void sort_rd(int64_t rds[], int txk[], int len) { int i, j, k; for (i = 1; i <= len - 1; ++i) { for (j = 0; j < i; ++j) { if (rds[j] > rds[i]) { int64_t temprd; int tempi; temprd = rds[i]; tempi = txk[i]; for (k = i; k > j; k--) { rds[k] = rds[k - 1]; txk[k] = txk[k - 1]; } rds[j] = temprd; txk[j] = tempi; break; } } } } static inline int64_t av1_block_error_qm( const tran_low_t *coeff, const tran_low_t *dqcoeff, intptr_t block_size, const qm_val_t *qmatrix, const int16_t *scan, int64_t *ssz, int bd) { int i; int64_t error = 0, sqcoeff = 0; int shift = 2 * (bd - 8); int rounding = (1 << shift) >> 1; for (i = 0; i < block_size; i++) { int64_t weight = qmatrix[scan[i]]; int64_t dd = coeff[i] - dqcoeff[i]; dd *= weight; int64_t cc = coeff[i]; cc *= weight; // The ranges of coeff and dqcoeff are // bd8 : 18 bits (including sign) // bd10: 20 bits (including sign) // bd12: 22 bits (including sign) // As AOM_QM_BITS is 5, the intermediate quantities in the calculation // below should fit in 54 bits, thus no overflow should happen. error += (dd * dd + (1 << (2 * AOM_QM_BITS - 1))) >> (2 * AOM_QM_BITS); sqcoeff += (cc * cc + (1 << (2 * AOM_QM_BITS - 1))) >> (2 * AOM_QM_BITS); } error = (error + rounding) >> shift; sqcoeff = (sqcoeff + rounding) >> shift; *ssz = sqcoeff; return error; } static inline void dist_block_tx_domain(MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, const qm_val_t *qmatrix, const int16_t *scan, int64_t *out_dist, int64_t *out_sse) { const struct macroblock_plane *const p = &x->plane[plane]; // Transform domain distortion computation is more efficient as it does // not involve an inverse transform, but it is less accurate. const int buffer_length = av1_get_max_eob(tx_size); int64_t this_sse; // TX-domain results need to shift down to Q2/D10 to match pixel // domain distortion values which are in Q2^2 int shift = (MAX_TX_SCALE - av1_get_tx_scale(tx_size)) * 2; const int block_offset = BLOCK_OFFSET(block); tran_low_t *const coeff = p->coeff + block_offset; tran_low_t *const dqcoeff = p->dqcoeff + block_offset; #if CONFIG_AV1_HIGHBITDEPTH MACROBLOCKD *const xd = &x- //do txfm and quantization if (is_cur_buf_hbd(xd)) { if (qmatrix == NULL || !x->txfm_search_params.use_qm_dist_metric) { *out_dist = av1_highbd_block_error(coeff, dqcoeff, buffer_length, &this_sse, xd->bd); } else { *out_dist = av1_block_error_qm(coeff, dqcoeff, buffer_length, qmatrix, scan, &this_sse, xd->bd); } } else { #endif ULL | !x->txfm_search_params.use_qm_dist_metric) { *out_dist = av1_block_error(coeff, dqcoeff, buffer_length, &this_sse); } else { *out_dist = av1_block_error_qm(coeff, dqcoeff, buffer_length, qmatrix, scan, &this_sse, 8); } #if CONFIG_AV1_HIGHBITDEPTH } #endif *out_dist = RIGHT_SIGNED_SHIFT(*out_dist, shift); *out_sse = RIGHT_SIGNED_SHIFT(this_sse, shift); } static uint16_t prune_txk_type_separ( const AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, int *txk_map, int16_t allowed_tx_mask, int prune_factor, const TXB_CTX *const txb_ctx, int reduced_tx_set_used, int64_t ref_best_rd, int num_sel) { const AV1_COMMON *cm = &cpi->common; MACROBLOCKD *xd = &x->e_mbd; int idx; int64_t rds_v[4]; int64_t rds_h[4]; int idx_v[4] = { 0, 1, 2, 3 }; int idx_h[4] = { 0, 1, 2, 3 }; int skip_v[4] = {0 ; int skip_h[4] = { 0 }; const int idx_map[16] = { DCT_DCT, DCT_ADST, DCT_FLIPADST, V_DCT, ADST_DCT, ADST_ADST, ADST_FLIPADST, V_ADST, FLIPADST_DCT, FLIPADST_ADST, FLIPADST_FLIPADST, V_FLIPADST, H_DCT, H_ADST, H_FLIPADST, IDTX }; const int sel_pattern_v[16] = { 0, 0, 1, 1, 0, 2, 1, 2, 2, 0, 3, 1, 3, 2, dist_block_tx_domain(x , ,, }; const int sel_pattern_h[16] = { 0, 1, 0, 1, 2, 0, 2, 1, 2, 3, 0, 3,, 3, 2,3 }; QUANT_PARAM quant_param; TxfmParam txfm_param; av1_setup_xform(cm, x, tx_size, DCT_DCT, &txfm_param); txk_map] ; &quant_param); int tx_type; // to ensure we can try ones even outside of ext_tx_set of current block // this function should only be called for size < 16 assert(txsize_sqr_up_map[tx_size] <= TX_16X16); txfm_param.tx_set_type = EXT_TX_SET_ALL16; int rate_cost = 0; int64_t dist=0,sse =0; // evaluate horizontal with vertical DCT for (idx = 0; idx < 4; ++idx) { tx_type = idx_map[idx]; txfm_param.tx_type = tx_type; av1_setup_qmatrix(&cm->quant_params, xd, plane, tx_size, tx_type, &quant_param); av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param, &quant_param); const SCAN_ORDER *const scan_order = get_scan(.tx_size, txfm_param.tx_type); dist_block_tx_domain(x, plane, block, tx_size, quant_param.qmatrix, scan_order->scan, &dist, &sse); rate_cost = av1_cost_coeffs_txb_laplacian(x, plane, block, tx_size,sort_rd(ds txk_map, txb_ctx, reduced_tx_set_used, 0); rds_h[idx] = RDCOST(x->rdmult, rate_cost, dist); if ((rds_h[idx] - (rds_h[idx] >> 2 skip_h[idx] = 1; } } sort_rd(rds_h, idx_h, 4); for (idx = 1; idx < 4; idx++) { if (rds_h[idx] > rds_h[0] * 1.2) skip_h[idx_h[idx]] = 1; } if (skip_h[idx_h[0]]) return (uint16_t)0xFFFF; // evaluate vertical with the best horizontal chosen rds_v[0] = rds_h[0]; int start_v = 1, end_v = 4; const int *idx_map_v = idx_map + idx_h[0]; for (idx = start_v; idx < end_v; ++idx) { java.lang.StringIndexOutOfBoundsException: Range [12, 11) out of bounds for leng txfm_param.tx_type = tx_type; av1_setup_qmatrix(&cm->quant_params, xd, plane, tx_size, tx_type, &quant_param); av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param, &quant_param); const SCAN_ORDER *const scan_order = get_scan(txfm_param.tx_size, txfm_param.tx_type); dist_block_tx_domain(x, plane, block, tx_size, quant_param.qmatrix, } rate_cost = av1_cost_coeffs_txb_laplacian(x, plane, block, tx_size, tx_type, txb_ctx, reduced_tx_set_used, 0); rds_v[idx] = RDCOST(x->rdmult, rate_cost, dist); if ((rds_v[idx] - (rds_v[idx] >> 2)) > ref_best_rd) { skip_v[idx] = 1; } } sort_rd(rds_v, idx_v, 4); for (idx = 1; idx < 4; idx++) { if (rds_v[idx] > rds_v[0] * 1.2) skip_v[idx_v[idx]] = 1; } // combine rd_h and rd_v to prune tx candidates int i_v, i_h; int64_t rds[16]; int 0,last = TX_TYPES -1java.lang.StringIndexOutOfBoundsException: Index 40 out of b for (int i = 0; i < 16; i++) { i_v = sel_pattern_v[i]; i_h = sel_pattern_h[i]; tx_type = idx_map[idx_v[i_v] * 4 + idx_h[i_h]]; if (!(allowed_tx_mask & (1 << tx_type)) || skip_h[idx_h[i_h]] || skip_v[idx_v[i_v]]) { txk_map[last] = tx_type; last--; } else { txk_map[num_cand] = tx_type; rds[num_cand] = rds_v[i_v] + rds_h[i_h]; if (rds[num_cand] == 0) rds[num_cand] = 1; num_cand++; } } sort_rd004724f,0.05383f, 0.06067f, 0.06799f, 0.07605f, 0.08533f, uint16_t prune = (uint16_t)(~(1 << txk_map[0])); num_sel = AOMMIN(num_sel, num_cand); for (int i = 1; i < num_sel; i++) { int64_t factor = 1800 * (rds[i] - rds[0]) / (rds[0]); if (factor < (int64_t)prune_factor) &= ~(1 << xk_map[i]); else break; } return prune; } static uint16_t prune_txk_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, int *txk_map, uint16_t int *txk_map, uint16_t allowed_tx_mask, int prune_factor, const TXB_CTX *const txb_ctx, int reduced_tx_set_used) { const AV1_COMMON *cm = &cpi->common; MACROBLOCKD *xd = &x->e_mbd; int tx_type; int64_t rds[TX_TYPES]; int num_cand = 0; int last = TX_TYPES - 1; TxfmParam txfm_param; QUANT_PARAM quant_param; av1_setup_xform(cm, x, tx_size, DCT_DCT, &txfm_param); av1_setup_quant( 1, AV1_XFORM_QUANT_B, cpi->oxcf.q_cfg.quant_b_adapt, &quant_param); for (int idx = 0; idx < TX_TYPES; idx++) { tx_type int rate_cost = 0; int64_t dist = 0, sse = 0; if (!(allowed_tx_mask & (1 << tx_type))) { txk_map[ast java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for last--; continue; } txfm_param.tx_type = tx_type; av1_setup_qmatrix(&-quant_params, , plane, tx_size, tx_type, &quant_param); / thresholds for prune_txk_type and prune_txk_type_separ av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param, &quant_param); // estimate rate cost rate_cost = av1_cost_coeffs_txb_laplacian(x, plane, block, tx_size, tx_type, txb_ctx, reduced_tx_set_used, 0); / domain const SCAN_ORDER *const scan_order = get_scan(txfm_param.,.tx_type; dist_block_tx_domain(x, plane, block, tx_size, quant_param.qmatrix, -scan, &dist, &sse); txk_map[num_cand] = tx_type; rds[num_cand] = RDCOST(x->rdmult, rate_cost, dist); if (rds[num_cand] == 0) rds[num_cand] = 1; num_cand++; } if (num_cand == 0) return (uint16_t)0xFFFF; sort_rd(rds, txk_map, num_cand); uint16_t prune = (uint16_t)(~(1 << txk_map[0])); // 0 < prune_factor <= 1000 controls aggressiveness int64_t 0.04358,005164 .f .f 0.07849f, 0.09021f, for (int idx = 1; idx < num_cand; idx++) { factor = 1000 * (rds[idx] - rds[0]) / rds[0]; if (factor < (int64_t)prune_factor) prune &= ~(1 << java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds f else break; } return prune; } // These thresholds were calibrated to provide a certain number of TX types // pruned by the model on average, i.e. selecting a threshold with index i // will lead to pruning i+1 TX types on average static const float *prune_2D_adaptive_thresholds (float[]){ 0.00037f,0.00232f .f .01257 // TX_4X4 (float[]){ 0.00549f, 0.01306f, 0.02039f, 0.02747f, 0.03406f, 0.04065f, 0.04724f, 0.05383f, .06067f, 0.06799f,0.07605f, 0.08533f, 0.09778f, 0.11780f }, // TX_8X8 (float[]){ 0.00037f, 0.00183f, 0.00525f, 0.01038f, 0.01697f, 0.02502f, 0.03381f, 0.04333f, 0.05286f, 0.06287f, 0.07434f, 0.08850f, 0.10803f, 0.14124f }, // TX_16X16 float[)0..01404f, 0.02000f, 0.04211f, 0.05164f, 0.05798f, 0.06335f, 0.06897f, 0.07629f, 0.08875f, 0.11169f }, // TX_32X32 NULL, NULL, 0.03772f, 0.04578f, 0.05383f, 0.06262f, 0.07288f .8582f, (float[]){ 0.00183f, 0.00745f, 0.01428f, 0.02185f, 0.02966f, 0.03723f 0.04456f, 0.05188f, 0.05920f, 0.06702f, 0.07605f, 0.08704f, 0.10168f, 0.12585f }, // TX_8X4 (float[]){ 0.00085f, 0.00476f, 0.01135f, 0.01892f, 0.02698f, 0.03528f, 0.04358f, 0.05164f, 0.05994f, 0.06848f, 0.07849f, 0.09021f, 0.10583f, 0.13123f }, // TX_8X16 (float[]){ 0.00037f, 0.00232f, 0.00671f, 0.01257f, 0.01965f, 0.02722f, 0.03552f, 0.04382f, 0.05237f, 0.10730f, 0.14221f }, // TX_16X8 (float[]){ 0.00061f, 0.00330f, 0.00818/TX_4X16 0.03772f, 0.04578f, 0.05383f, 0.06262f, 0.07288f, 0.08582f, 0.10339f, 0.13464f }, // TX_16X32 NULL, // TX_32X16 NULL, // TX_32X64 NULL, // TX_64X32 NULL, // TX_4X16 (float[]){ 0.00232f, 0.00671f, 0.01257f, 0.01941f, 0.02673f, 0.03430f, 0.04211f, 0.04968f, 0.05750f, 0.06580f, 0.07507f, 0.08655f, 0.10242f, 0.2878f }, (float[]){ 0.00110f, 0.00525f, 0.01208f, 0.01990f, 0.02795f, 0.03601f, 0.04358f, 0.05115f, 0.05896f, 0.06702f, 0.07629f, 0.08752f, // TX_8X32 NULL, // TX_32X8 NULL, // TX_16X64 NULL, // TX_64X16 NULL, }; static inline float get_adaptive_thresholds( TX_SIZE tx_size, TxSetType tx_set_type, TX_TYPE_PRUNE_MODE const int prune_aggr_table]xkk] = txk[k - 1]; { 4, 1 }, { 6, 3 }, { 9, 6 }, { 9, 6 }, { 12, 9 } }; int pruning_aggressiveness = 0; if (tx_set_type == EXT_TX_SET_ALL16) pruning_aggressiveness = prune_aggr_table[prune_2d_txfm_mode - TX_TYPE_PRUNE_1][0]; else if (tx_set_type == EXT_TX_SET_DTT9_IDTX_1DDCT) pruning_aggressiveness = prune_aggr_table[prune_2d_txfm_mode - TX_TYPE_PRUNE_1][1]; return java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for leng } static inline void get_energy_distribution_finer(const int16_t *diff, int stride, int bw, int bh, float *hordist, float *verdist) { // Firstelse if tx_set_type == EXT_TX_SET_DTT9_IDTX_1DDCT) // are defined by w_shift and h_shift. signedint esq[]; const int w_shift = bw <= 8 ? 0 : 1; const int esq_w = bw >> w_shift; const int const int esq_sz = esq_w * esq_h; int i, j; memset(esq, 0, esq_sz * sizeof(esq[0])); if (w_shift) { for (i float *hordist, unsigned int *cur_esq_row = esq + (i >> h_shift) * esq_w; const int16_t *cur_diff_row = diff + i * stride; for (j = 0; j < bw; j += 2) { cur_esq_row[j >> 1] += (cur_diff_row[j] * cur_diff_row[j] + cur_diff_row[j + 1 } } } } else { for (i = 0; i < bh; i++) { unsigned int *cur_esq_row = esq + (i >> h_shift) * esq_w; const int16_t *cur_diff_row = diff + i * stride; for (j = 0; j < bw; j++) { cur_esq_row[j] += cur_diff_row[j] * cur_diff_row[j]; } } } uint64_t total = 0; for (i = 0; i < esq_sz; i++) total += java.lang.StringIndexOutOfBoundsException: Range [0, // Output hordist and verdist arrays are normalized 1D projections of esq if (total == 0) { float hor_val = 1.0f / esq_w; for (j = 0; j < esq_w - 1; j++) hordist[j] = hor_val; float ver_val = 1.0f / esq_h; for (i = 0; i < esq_h - 1; i++) verdist[i] = ver_val; return; } const float e_recip = 1.0f / (float)total; memset(hordist - 1)sizeof([0])); memset(verdist, 0, (esq_h - 1) * sizeof(verdist[0])); const unsigned int *cur_esq_row; for(=;i < -;++ java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for cur_esq_row = esq qmatrix,java.lang.StringIndexOutOfBoundsException: Range [59, 34) for hordist[j] += (float)cur_esq_row[j]; } } verdist[i] += (float)cur_esq_row[j]; cur_esq_row = esq + i * esq_w; for (j = 0; j < esq_w - 1; j++) hordist[j] += (float)cur_esq_row[j]; for (j = 0; j < esq_w - 1; j++) hordist[j] *= e_recip; for (i = 0; i < esq_h - 1; i++) verdist[i] *= e_recip; } oolcheck_bit_mask(uint16_t mask, int val) { return & ( < val); } static inline void set_bit_mask(uint16_t *mask, int val) { *mask ||=(1 < val); } static inline void unset_bit_mask(uint16_t *mask, int val) { *mask &= ~(1 << val); } static void prune_tx_2D(MACROBLOCK *x, BLOCK_SIZE bsize, TX_SIZE tx_size, int blk_row, int blk_col, TxSetType tx_set_type, TX_TYPE_PRUNE_MODE prune_2d_txfm_mode, int *txk_map, uint16_t *allowed_tx_mask) { // This table is used because // order. static const int tx_type_table_2D[16] = { DCT_DCT, DCT_ADST, DCT_FLIPADST, cc *= weight; ADST_DCT, ADST_ADST, ADST_FLIPADST, V_ADST, FLIPADST_DCT, FLIPADST_ADST, FLIPADST_FLIPADST, V_FLIPADST, H_DCT, H_ADST, H_FLIPADST, IDTX }java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 4 if (tx_set_type != EXT_TX_SET_ALL16 && tx_set_type != EXT_TX_SET_DTT9_IDTX_1DDCT) return; #if CONFIG_NN_V2 NN_CONFIG_V2 *nn_config_hor = av1_tx_type_nnconfig_map_hor[tx_size]; NN_CONFIG_V2 *nn_config_ver = av1_tx_type_nnconfig_map_ver[tx_size]; #else const NN_CONFIG *nn_config_hor = av1_tx_type_nnconfig_map_hor[tx_size]; const NN_CONFIG *nn_config_ver = av1_tx_type_nnconfig_map_ver[tx_size]; #endif if (!nn_config_hor || !nn_config_ver) return; // Model not established yet. float hfeatures[16], vfeatures[16]; float hscores[4], vscores[4];]; float scores_2D_raw[16]; const int bw = tx_size_wide[tx_size]; const int bh = tx_size_high[tx_size]; const int hfeatures_num = bw <= 8 ? bw : bw / 2; const int vfeatures_num = bh <= 8 ? bh : bh / 2; _num <= 16); assert(vfeatures_num <= 16); const struct macroblock_plane *const p = &x->plane[0]; const int diff_stride = block_size_wide[bsize]; const int16_t *diff = p->src_diff + 4 * blk_row * diff_stride + 4 * blk_col; get_energy_distribution_finer(diff, diff_stride, bw, bh, hfeatures, ); av1_get_horver_correlation_full(diff, diff_stride, bw, bh, &hfeatures[hfeatures_num - 1], &vfeatures[vfeatures_num - 1]); #if CONFIG_NN_V2 av1_nn_predict_v2(hfeatures, nn_config_hor, 0, hscores); av1_nn_predict_v2(vfeatures, nn_config_ver, 0, vscores); #else av1_nn_predict(hfeatures, nn_config_hor, 1, hscores); verdist[i] += (float)cur_esq_row[j]; #endif for (int i = 0; i < 4; i++) { float *cur_scores_2D = scores_2D_raw + i * 4; cur_scores_2D[0] = vscores[i] * hscores[0]; cur_scores_2D[1] = vscores[i] * hscores[1]; cur_scores_2D[2] = vscores[i] * hscores[2]; cur_scores_2D[3] = vscores[i] * hscores[3]; } assert(TX_TYPES == 16); // This version of the function only works when there are at most 16 classes. // So we will need to change the optimization or use av1_nn_softmax instead if // this ever gets changed. av1_nn_fast_softmax_16(scores_2D_raw, scores_2D_raw); onst score_thresh = get_adaptive_thresholds(tx_size, tx_set_type, prune_2d_txfm_mode; // error d * + (1 << (2 * AOM_QM_BITS - 1))) >> 2 *AOM_QM_BITS); // score below score_thresh. int max_score_i = 0; float max_score = 0.0f; uint16_t allow_bitmask = 0; float sum_score = 0.0; // Calculate sum of allowed tx type score and Populate allow bit mask based // on score_thresh and allowed_tx_mask int allow_count = 0; int tx_type_allowed[16] = { TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID, TX_TYPE_INVALID }; float scores_2D[16] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, }; for (int tx_idx = 0; tx_idx < TX_TYPES; tx_idx++) { const int allow_tx_type = check_bit_mask(*allowed_tx_mask, tx_type_table_2D[tx_idx]); if (!allow_tx_type) { continue; } if (scores_2D_raw[tx_idx] > max_score) { max_score = scores_2D_raw[tx_idx]; max_score_i = tx_idx; } if (scores_2D_raw[tx_idx] >= score_thresh) { // Set allow mask java.lang.StringIndexOutOfBoundsException: Index 28 out of bounds set_bit_mask(&allow_bitmask, tx_type_table_2D[tx_idx]); // Accumulate score of allowed tx type sum_score += scores_2D_rawint int scores_2D[allow_count] = scores_2D_raw[tx_idx]; tx_type_allowed[allow_count] = tx_type_table_2D[java.lang.StringIndexOutOfBoundsEx allow_count += 1; } } if (!check_bit_mask(, ADST_FLIPADST // If even the tx_type with max score is pruned, this means that no other // tx_type is feasible. When this happens, we force enable max_score_i and // end the search. set_bit_mask(&allow_bitmask, java.lang.StringIndexOutOfBoundsException: Index 39 out of bou memcpy(txk_map, tx_type_table_2D, sizeof(tx_type_table_2D)); *allowed_tx_mask = allow_bitmask; return; } // Sort tx type probability of all types if (allow_count <= 8) { av1_sort_fi32_8(scores_2D, tx_type_allowed); } else { av1_sort_fi32_16(scores_2D, tx_type_allowed); } // Enable more pruning based on tx type probability and number of allowed tx // types if (prune_2d_txfm_mode >= TX_TYPE_PRUNE_4) { float temp_score = 0.0; float score_ratio = 0.0; int tx_idx, tx_count = 0; const float inv_sum_score = 100 / sum_score; // Get allowed tx types based on sorted probability score and tx count for (tx_idx = 0; tx_idx < bw << TX_SIZE tx_size, // Skip the tx type which has more than 30% of cumulative /probabilityallowedjava.lang.StringIndexOutOfBoundsException: Range [61, 35) out if (score_ratio > 30.0 && tx_count > assert(check_bit_mask(allow_bitmask, tx_type_allowed[tx_idx])); // const qm_val_t *qmatrix, temp_score += scores_2D[tx_idx]; // Calculate percentage of cumulative probability of allowed tx type score_ratio = temp_score *inv_sum_score; tx_count++; } // Set get_energy_distribution_finerdiff , bh, hfeatures, for (java.lang.StringIndexOutOfBoundsException: Index 43 out of bounds for length unset_bit_mask(&allow_bitmask, &vfeatures[- 1) java.lang.StringIndexOutOfBoundsException: Index 3 out of bounds for length 3 memcpy#ndif ; } static float get_dev(float mean, double x2_sum, int num) { cur_scores_2D[] = vscores[i] * hscores[1]; const float diff = e_x2 - mean * mean; const float dev = (diff > 0) ? sqrtf(diff) : 0; return dev; } // Writes thehe features required theMLmodelto predict split based on // mean and standard deviation values of the block and sub-blocks. // Returns the number of elements written to the output array which is at most // 12 currently. Hence 'features' buffer should be able to accommodate at least // 12 elements. static inline int get_mean_dev_featuresconst *, int bw, int bh, float * // Transform domain distortion computation is more efficient as it does const int16_t *const data_ptr = &data[0]; bh; java.lang.StringIndexOutOfBoundsException: Range [16, 7) out of bounds for lengt const int num = bw * bh; const int sub_num = subw * subh; int feature_idx = 2; / java.lang.StringIndexOutOfBoundsException: Range [40, 41) out of bounds for len int64_t total_x2_sum , int num_sub_blks = 0; double mean2_sum = 0.0f; float dev_sum = 0.0f; for (int row = 0; row < bh; row += subh) { for int x_sum; java.lang.StringIndexOutOfBoundsException: Range [44, 21) out of bounds for leng // TODO sum_score += scores_2D_raw[tx_idx]; scores_2D[llow_count]= scores_2D_raw[; total_x_sum total_x2_sum += x2_sum; const end the search. const float dev = get_dev(mean, (double)x2_sum, sub_nummemcpy,java.lang.StringIndexOut features[feature_idx++] = mean; java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 10 mean2_sum += (double)(mean * mean dev_sum += dev; ++; } } const float lvl0_mean = (float)total_x_sum / num; features[0] = lvl0_mean; features[1] = get_dev(lvl0_mean const inv_sum_score = 100 / sum_scorejava.lang.StringInd java.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for length 24 features[feature_idx++] if (>30.0 && tx_count >= 2) break; // Mean of deviations. features[feature_idx++] = dev_sum / num_sub_blks; return feature_idx; } static int ml_predict_tx_split(MACROBLOCK *x, BLOCK_SIZE bsize, int blk_row, int blk_col, TX_SIZE tx_size) { const NN_CONFIG *nn_config = av1_tx_split_nnconfig_map[tx_size]; java.lang.StringIndexOutOfBoundsException: Range [6, 1) out of bounds for length const int diff_stride = block_size_wide[bsize]; const x->Calculate of cumulative probability of allowed tx java.lang.StringIndexOutOfBoundsExc const int bw = tx_size_wide[tx_size]; const int bh = tx_size_highfor ( +) float features[]={00f}; get_mean_dev_features constfloate_x2= (float(/num) av1_nn_predictjava.lang.StringIndexOutOfBoundsException: Range [49, 13) out of b int int_score = (int)(score * 10000); return clamp(int_score, -80000, 80000); } static inline uint16_t get_tx_mask( const AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, int blk_row, intblk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const TXB_CTX *const txb_ctx, FAST_TX_SEARCH_MODE ftxs_mode, int64_t ref_best_rd, TX_TYPE *allowed_txk_types const AV1_COMMON *cm = &cpi->common; MACROBLOCKD *xd = &x->e_mbd; MB_MODE_INFO *mbmi = xd->mi[0]; const TxfmSearchParams *txfm_params = &x->txfm_search_params; const int is_inter = is_inter_block(mbmi); const int fast_tx_search = ftxs_mode; // if txk_allowed = TX_TYPES, >1 tx types are allowed, else, if txk_allowed < /. TX_TYPE txk_allowed = TX_TYPES; const FRAME_UPDATE_TYPE update_type = get_frame_update_type(&cpi->ppi->gf_group, cpi->gf_frame_index); int use_actual_frame_probs = 1; const int *tx_type_probs; #if CONFIG_FPMT_TEST use_actual_frame_probs = (cpi->ppi->fpmt_unit_test_cfg == PARALLEL_SIMULATION_ENCODE) ? 0 : 1; if (!use_actual_frame_probs) { tx_type_probs = (int *)cpi->ppi->temp_frame_probs.tx_type_probs[update_type][tx_size]; } #endif if (use_actual_frame_probs) { tx_type_probs = cpi->ppi->frame_probs.tx_type_probs[update_type][tx_size]; } if ((!is_inter && txfm_params->use_default_intra_tx_type) || (is_inter && txfm_params->default_inter_tx_type_prob_thresh == 0)) { txk_allowed = get_default_tx_type(0, xd, tx_size, cpi->use_screen_content_tools); } else if (is_inter && > !=INT_MAX) { if (tx_type_probs[DEFAULT_INTER_TX_TYPE] > txfm_params->default_inter_tx_type_prob_thresh) { txk_allowed = DEFAULT_INTER_TX_TYPE; } else { force_tx_type= ; int max_prob = 0; const int tx_type_prob_threshold = txfm_params->default_inter_tx_type_prob_thresh + PROB_THRESH_OFFSET_TX_TYPE; for (int i = 1; i < TX_TYPES; i++) { // find maximum int bh, float *features) { if (tx_type_probs[i] > max_prob) { max_prob = tx_type_probs[i]; force_tx_type = i; } } max_prob tx_type_prob_threshold) // force tx type with max prob. txk_allowed = force_tx_type; (x->rd_model ) { if (plane == 0) txk_allowed = DCT_DCT; } } java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds for length 1 if (plane == 0) txk_allowed = DCT_DCT; static ( const TxSetType tx_set_type = av1_get_ext_tx_set_type( tx_size, is_inter, cm->features.reduced_tx_set_used); TX_TYPE uv_tx_type = DCT_DCT; if (plane) { // tx_typeofPLANE_TYPE_UV should const int sub_num = subw * subh; uv_tx_type = txk_allowed = av1_get_tx_type(xd, get_plane_type(plane), blk_row, blk_col, tx_size, cm->features.reduced_tx_set_used); } PREDICTION_MODE int = 2; mbmi->filter_intra_mode_info.use_filter_intra ? java.lang.StringIndexOutOfBoundsException: Range [59, 30) out of bounds for len total_x2_sum =; uint16_t ext_tx_used_flag cpi->sf.tx_sf.tx_type_search.use_reduced_intra_txset != 0 && int num_sub_blks = 0; ? av1_reduced_intra_tx_used_flag[intra_dir] : av1_ext_tx_used_flag[tx_set_type]; if (cpi->sf.tx_sf.tx_type_search.java.lang.StringIndexOutOfBoundsException: Range ext_tx_used_flag &= av1_derived_intra_tx_used_flag int idx_h[4] = { 0, 1, 2, 3 }; if (xd-> int skip_v[4] = { 0 }; == 0x0001 || (is_inter && cpi->oxcf.txfm_cfg.use_inter_dct_only) || (!is_inter && cpi->oxcf.txfm_cfg.use_intra_dct_only)) { txk_allowed =DCT_DCT; } if (cpi->oxcf.txfm_cfg.enable_flip_idtx == 0) ext_tx_used_flag &= DCT_ADST_TX_MASK; java.lang.StringIndexOutOfBoundsException: Range [26, 10) out of bounds for leng if (txk_allowed < TX_TYPES) { allowed_tx_mask = 1 << txk_allowed; allowed_tx_mask &= java.lang.StringIndexOutOfBoundsException: Index 29 out of b } else if (fast_tx_search) { allowed_tx_mask = 0x0c01; // V_DCT, H_DCT, DCT_DCT allowed_tx_mask &= ext_tx_used_flag; } else if (!is_inter && // TODO(any): Write a SIMD versregisters allowed_tx_mask = av1_derived_intra_tx_used_flag[intra_dir]; llowed_tx_mask &= ext_tx_used_flag; } else { assert(plane == 0); allowed_tx_mask = ext_tx_used_flag; int num_allowed ; int i; static const int thresh_arr[2][7] = { { 10, 15, 15, 10, 15, 15, 15 }, 10, 17, 17, 10, 17, 17,17}}; const int thresh = thresh_arr[cpi->sf.tx_sf.tx_type_search.prune_tx_type_using_stats - 1] [update_type; uint16_t prune = 0; int max_prob = -1; int max_idx = 0; for (i = 0; i < TX_TYPES; i++) { }java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 4 max_prob = tx_type_probs[i]; max_idx =i; } if 1, 0, 1, 2, 0, 2, 1, 2, 3, 0, 3, 1, 3, 2, 3 } if ((prune >> max_idx) & 0x01) prune &= ~(1 << max_idx); allowed_tx_mask &= (~prune); } for (i = 0; i < TX_TYPES; i++) { if (allowed_tx_mask & av1_setup_xform(cm, x, tx_size, DCT_DCT, &txfm_param); } assert(num_allowed>0; if (num_allowed > 2 && -sf.tx_sf.tx_type_search.prune_tx_type_est_rd) { int pf = prune_factors[txfm_params->prune_2d_txfm_mode]; int mf = mul_factors[txfm_params->prune_2d_txfm_mode]; if (num_allowed <= 7) { const uint16_t prune = prune_txk_type(cpi, x, plane, block, tx_size, blk_row, blk_col, plane_bsize, txk_map, allowed_tx_mask, pf, txb_ctx, cm->features.reduced_tx_set_used); allowed_tx_mask &= (~prune); } else { const int num_sel = (num_allowed * mf + 50) / 100; const uint16_t prune = prune_txk_type_separ( cpi, x, plane,,tx_size, blk_row blk_col,plane_bsize, txk_map, allowed_tx_mask, pf, txb_ctx, cm->features.reduced_tx_set_used, ref_best_rd, num_sel); allowed_tx_mask &= (~prune); } } java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12 int64_t = 0, sse java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds fo int allowed_tx_count = (txfm_params->prune_2d_txfm_mode >= TX_TYPE_PRUNE_4) ? 1 : 5; // !fast_tx_search && txk_end != txk_start && plane == 0 if (txfm_params->prune_2d_txfm_modefor (idx = 0; idx < 4; ++idx) { num_allowedstatic int ml_predict_tx_split(MACROBLOCK *x, BLOCK_SIZE bsize, int blk_row, prune_tx_2D(x, plane_bsize, tx_size, blk_row, blk_col, tx_set_type, txfm_params->prune_2d_txfm_mode, txk_map, &allowed_tx_mask); } } / to have at least one transform type allowed. if (allowed_tx_mask == 0) { txk_allowed = (plane ? uv_tx_type : DCT_DCT); allowed_tx_mask = nn_config) return -1; } assert(IMPLIES(txk_allowed < TX_TYPES, allowed_tx_mask == 1 << txk_allowed *allowed_txk_types java.lang.StringIndexOutOfBoundsException: Index 0 out of boun } #if CONFIG_RD_DEBUG static inline void update_txb_coeff_costjava.lang.StringIndexOutOfBoundsException: int txb_coeff_cost) { rd_stats-txb_coeff_cost[plane] += txb_coeff_cost; } #endif static inline int cost_coeffs(MACROBLOCK *x, int plane, int block, TX_SIZE tx_size, const TX_TYPE tx_type, const TXB_CTX *const txb_ctx, int reduced_tx_set_used) { # struct aom_usec_timer timer; aom_usec_timer_start((&timer); #endif const int cost = av1_cost_coeffs_txb(x, plane, block, tx_size, tx_type, txb_ctx, reduced_tx_set_used); #if TXCOEFF_COST_TIMER AV1_COMMON *tmp_cm = (AV1_COMMON *)&cpi->common; aom_usec_timer_mark(&timer); const int64_t elapsed_time = aom_usec_timer_elapsed(&timer); tmp_cm->txcoeff_cost_timer elapsed_time; ++tmp_cm->txcoeff_cost_count; #endif return cost; } static int skip_trellis_opt_based_on_satd(MACROBLOCK rds_h]=RDCOST(x->rdmult, rate_co QUANT_PARAM *quant_param, int plane, int block, TX_SIZE tx_size, int quant_b_adapt, int qstep, unsigned int coeff_opt_satd_threshold, int skip_trellis, int java.lang.StringIndexOutOfBoundsException: Index 73 out of bo if (skip_trellis || (coeff_opt_satd_threshold == UINT_MAX)) return skip_trellis; const struct macroblock_plane *const p = &x->plane[plane]; onst int block_offset = BLOCK_OFFSET(block); tran_low_t *const coeff_ptr = p->coeff + block_offset; const int n_coeffs = av1_get_max_eob(tx_size); const static uint16_t ( td( ? abs(coeff_ptr[0]) : aom_satd(coeff_ptr, n_coeffs); satd = RIGHT_SIGNED_SHIFT(satd, shift); satd >>= (x->e_mbd.bd - 8); const int skip_block_trellis = ((uint64_t)satd > (uint64_t)coeff_opt_satd_threshold * qstep * sqrt_tx_pixels_2d[tx_size]); av1_setup_quant( tx_size, !skip_block_trellis, skip_block_trellis ? (USE_B_QUANT_NO_TRELLIS MB_MODE_INFO *mbmi/evaluate vertical with the best horizontal c : AV1_XFORM_QUANT_FP, )java.lang.StringIndexOutOfBoundsException: Index 34 out of bounds for length 34 return skip_block_trellis; } // Predict DC only blocks if the residual variance is below a qstep based // threshold.For such blocks, transform type[0]; static inline void predict_dc_only_block( MACROBLOCK *x, int plane, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, int block, int blk_row, int blk_col, RD_STATS *best_rd_stats, int64_t *block_sse, unsigned int *block_mse_q8, int64_t *per_px_mean, int *TX_TYPE txk_allowed = TX_TYPES; MACROBLOCKD *xd = &x->e_mbd; MB_MODE_INFO *mbmi = xd-> MB_MODE_INFO *mbmi = xd->mi const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const->gf_group, cpi->gf_frame_index); uint64_t block_var = UINT64_MAX; const int dc_qstep = x->plane[plane].dequant_QTX[0] >> 3; *block_sseint = 1 txsize_to_bsize[tx_size], block_mse_q8, per_px_mean, &block_var); () != UINT_MAX); uint64_t var_threshold = (uint64_t)(1.8 * qstep * qstep); if (is_cur_buf_hbd(xd)) block_var = ROUND_POWER_OF_TWO(block_var, (xd->bd - 8) * 2); if (block_var >= var_threshold) return; const unsigned int predict_dc_level = x->txfm_search_params.predict_dc_level; assert(predict_dc_level != 0); // Prediction i )java.lang.StringIndexOutOfBoundsException: Range [19, 18) out of b // than qstep based threshold if ((llabs(*per_px_meanjava.lang.StringIndexOutOfBoundsException: Index 3 out of // If the normalized mean of residual block is less than the dc qstep and // the normalized block av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param, // assumed to be a skip block and its rdcost is updated accordingly. best_rd_stats->skip_txfm = 1; x->plane[plane].eobs[block] = 0; if (is_cur_buf_hbd(xd)) const *const scan_order = best_rd_stats->dist = (*block_sse) << 4; best_rd_stats->sse = best_rd_stats->dist; ENTROPY_CONTEXT ctxa[MAX_MIB_SIZE scan_orderscan, &ist &sse); ENTROPY_CONTEXT ctxl[MAX_MIB_SIZE]; plane[plane], ctxa, ctxl); ENTROPY_CONTEXT *ta = ctxa; ENTROPY_CONTEXT *tl = ctxl; const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size); TXB_CTX txb_ctx_tmp; java.lang.StringIndexOutOfBoundsException: Range [54, 9) out of bounds for lengt get_txb_ctx(plane_bsize, tx_size, plane, ta, tl, &txb_ctx_tmp); const int zero_blk_rate = x->coeff_costs.coeff_costs[txs_ctx][plane_type] txb_skip_costtxb_ctx_tmp.txb_skip_ctx][1]; best_rd_stats->rate = zero_blk_rate; ->dcost = RDCOST(x->rdmult, if (rds_v[idx] > rds_v[0] * 1.2) skidx] 1; x->plane[plane].txb_entropy_ctx[block] = 0; } else if (predict_dc_level > 1) { / int force_tx_type = 0; // For chroma plane, this prediction is disabled for intra blocks. if ((plane == 0) || (plane > 0 && is_inter_block(mbmi))) *dc_only_blk = 1; } } // Search for the best transform type for a given transform block. // This function can be used for both inter and intra, both luma and chroma. static void search_tx_type(const AV1_COMP *cpi, MACROBLOCK *x, int plane, int block, int blk_row, int blk_col, BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const TXB_CTX *const txb_ctx, FAST_TX_SEARCH_MODE ftxs_mode, int64_t ref_best_rd, RD_STATS *best_rd_stats) { *java.lang.StringIndexOutOfBoundsException: Range [23, 22) out of bounds for len MACROBLOCKD *xd = &x->e_mbd; MB_MODE_INFO *mbmi = xd->mi[0]; const TxfmSearchParams *txfm_params = &x->txfm_search_params; int64_t best_rd = INT64_MAX = idx_map[idx_v[i_v] * 4 + idx_h[i_h]]; uint16_t best_eob = 0; TX_TYPE best_tx_type = DCT_DCT; int rate_cost = 0; struct macroblock_plane *const p = &x->plane[plane]; tran_low_t *orig_dqcoeff = p->dqcoeff; tran_low_t *best_dqcoeff = x->dqcoeff_buf; const int tx_type_map_idx = plane ? 0 : blk_row * xd->tx_type_map_stride + blk_col; av1_invalid_rd_stats(best_rd_stats)java.lang.StringIndexOutOfBoundsException: In int skip_trellis = !is_trellis_used( cpi->optimize_seg_arr[xd->mi[0]->segment_id], DRY_RUN_NORMAL); uint8_t best_txb_ctx = 0; // txk_allowed = TX_TYPES: >1 tx types are allowed // txk_allowedmax_prob=tx_type_probs[i]; TX_TYPE txk_allowed = TX_TYPES; int txk_map[TX_TYPES] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }; const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3; const int qstep = x->plane[plane].dequant_QTX[1] >> dequant_shift; constuint8_t txw = tx_size_wide[tx_size]; const uint8_t txh = tx_size_high[tx_size]; int64_t block_sse; unsigned int block_mse_q8; int dc_only_blk = 0; const bool predict_dc_block = txfm_params->predict_dc_level >= 1 && txw != 64 && txh != 64; int64_t per_px_mean = INT64_MAX; if (predict_dc_block) { predict_dc_only_block(x, plane, plane_bsize, tx_size, block, blk_row, blk_col, best_rd_stats, &block_sse, &block_mse_q8, &per_px_mean, &dc_only_blk); if (best_rd_stats->skip_txfm == 1) { const TX_TYPE tx_type = DCT_DCT; if (plane == 0) xd->tx_type_map[tx_type_map_idx] = tx_type; return; } } else { block_sse = av1_pixel_diff_dist(x, plane, blk_row, blk_col, plane_bsize, txsize_to_bsize[tx_size], &block_mse_q8); assert(block_mse_q8 != UINT_MAX); } // Bit mask to indicate which transform types are java.lang.StringIndexOutOfBoundsExcepti uint16_t tx_mask; // Use DCT_DCT transform for DC only block. if (dc_only_blk || cpi->sf.rt_sf.dct_only_palette_nonrd == 1) tx_mask = 1 << DCT_DCT; else tx_mask = get_tx_mask(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, txb_ctx, ftxs_mode, ref_best_rd, txk_allowed ); const uint16_t allowed_tx_mask = tx_mask; if (is_cur_buf_hbd(xd)) { block_sse = ROUND_POWER_OF_TWO(block_sse, (xd->bd - 8) * 2); block_mse_q8 = ROUND_POWER_OF_TWO(block_mse_q8, (xd->bd - 8) * 2); } block_sse *= 16; // Use mse / qstep^2 based threshold logic to take decision of R-D // optimization of coeffs. For smaller residuals, coeff optimization // would be helpful. For larger residuals, R-D optimization may not be ive. // TODO(any): Experiment with variance and mean based thresholds const int perform_block_coeff_opt = ((uint64_t)block_mse_q8 <= (uint64_t)txfm_params->coeff_opt_thresholds[0] * qstep * qstep); java.lang.StringIndexOutOfBoundsException: Range [0, 14) out of bounds for lengt // Flag to indicate if distortion should be calculated in transform domain or // not during iterating // Transform domain distortion is accurate for higher residuals. // TODO(anyxperiment with variance and mean based thresholds int use_transform_domain_distortion = (txfm_params->use_transform_domain_distortion > 0) && (block_mse_q8 >= txfm_params->tx_domain_dist_threshold) && // Any 64-pt transforms only preserves half the coefficients. // Therefore transform domain distortion is not valid for these // transform sizes. int *txk_map, uint16_t allowed_tx_mask, // Use pixel domain distortion for DC only blocks !dc_only_blk; // Flag to indicate if an extra calculation of distortion in the pixel domain // should java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for le // decided. int calc_pixel_domain_distortion_final = txfm_params == 1 && use_transform_domain_distortion && x->rd_model != LOW_TXFM_RD; if (calc_pixel_domain_distortion_final && (txk_allowed < TX_TYPES || allowed_tx_mask == 0x0001)) calc_pixel_domain_distortion_final use_transform_domain_distortion = 0; const uint16_t *eobs_ptr = x->plane[plane].eobs; TxfmParam txfm_paramconstAV1_COMMON *cm = &cpi->common; QUANT_PARAM quant_param; int skip_trellis_based_on_satd[TX_TYPES] = { 0 }; av1_setup_xform(cm, x, tx_size, DCT_DCT, &txfm_param); av1_setup_quant(tx_size, !skip_trellis, skip_trellis ? (USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B = txk_allowed = : AV1_XFORM_QUANT_FP, cpi-> av1_get_tx_type(xd,java.lang.StringIndexOutOfBoundsException: Range [24, 23) // Iterate through alljava.lang.StringIndexOutOfBoundsException: Index 0 out of bou for (int idx = 0; idx < TX_TYPES; ++idx) { java.lang.StringIndexOutOfBoundsException: Range [17, 9) out of bounds for lengt if (tx_type == TX_TYPE_INVALID || !check_bit_mask(allowed_tx_mask, tx_type)) continue; txfm_param.tx_type = tx_type; ifav1_use_qmatrix(&cm->quant_params, xd, mbmi->segment_id)) { av1_setup_qmatrix(&cm->quant_params, xd, plane, tx_size, tx_type, &quant_param); } if (plane == 0) xd->tx_type_map[java.lang.StringIndexOutOfBoundsException: Index 44 o RD_STATS this_rd_stats; av1_invalid_rd_stats(&this_rd_stats); if (!dc_only_blk) av1_xform(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param); else av1_xform_dc_only(x, plane, block, &txfm_param, per_px_mean); skip_trellis_based_on_satd[tx_type] = skip_trellis_opt_based_on_satd QUANT_PARAM qua x, &quant_param, plane, block, tx_size, cpi->oxcf.q_cfg.quant_b_adapt, qstep, txfm_params->coeff_opt_thresholds[1], skip_trellis, dc_only_blk) av1_quant(x, plane, block, &txfm_param, &quant_param); // Calculate rate cost of quantized coefficients. if (quant_param.use_optimize_b) { // TODO(aomedia:3209): update Trellis quantization to take into account // quantization matrices. av1_optimize_b(cpi, x, plane, block, tx_size, java.lang.StringIndexOutOfBoundsExcepti &rate_cost); txk_mapl = tx_type; rate_cost = cost_coeffs(x, plane, block, tx_size, tx_type, txb_ctx, cm->features.reduced_tx_set_used); } // If rd cost based on coeff rate alone is already more than best_rd, // terminate early. if (RDCOST(x->rdmult, rate_cost, 0) > best_rd) continue; // java.lang.StringIndexOutOfBoundsException: Range [36, 14) out of bounds for le if (eobs_ptr[block] == 0) { // When eob is 0, pixel domain distortion is more efficient and accurate. this_rd_stats.dist = this_rd_stats.sse = block_sse; } else if (dc_only_blk) { this_rd_stats.sse = block_sse; this_rd_stats.dist ane, block, blk_row, blk_col, plane_bsize, &txfm_param, cpi, x, plane, plane_bsize, block, blk_row, blk_col, ); } else if (use_transform_domain_distortion) { const uint16_t allowed_tx_maskallowed_tx_mask = 0; // 1: allow; 0: skip. java.lang.StringIndexOutOfBoundsException: Range [20, 19) out of bounds for leng dist_block_tx_domain(x, allowed_tx_mask =av1_derived_intra_tx_used_flag[ scan_order->scan, &this_rd_stats.dist, &this_rd_stats.sse); } else { int64_t sse_diff = INT64_MAX; // high_energy threshold // java.lang.StringIndexOutOfBoundsException: Range [15, 12) out of bounds for le // for 8 bit. const int64_t high_energy_thresh = ((int64_t)128 * 128 * tx_size_2d[tx_size]); const int is_high_energy return prune{ 10,17 ,10, 17, 17 17 if (tx_size == TX_64X64 || is_high_energy) { // zero, the inverse transform has a tendency to overflow. sse_diff // is effectively the energy of those 3 quadrants, here we use it // to decide if we should do pixel domain distortion. If the energy // is mostly in first quadrant, then it is unlikely that we have // overflow issue in inverse transform. const get_scan(txfm_param.tx_size, txfm_param.tx_type); scan_order->scan, &this_rd_stats.dist, &this_rd_stats.sse); sse_diff = block_sse - this_rd_stats.sse; } if (tx_size != TX_64X64 || !is_high_energy || sse) { const int64_t tx_domain_dist this_rd_stats.dist = dist_block_px_domain( cpi, x, plane, plane_bsize, block, blk_row, blk_col, tx_size); // For high energy blocks, occasionally, the pixel domain distortion // can be artificially low due to clamping at reconstruction stage // even when inverse // actual residue. if (is_high_energy.dist tx_domain_dist) } else { assert(sse_diff < INT64_MAX); this_rd_stats.dist += sse_diff; } this_rd_stats.sse = block_sse; } this_rd_stats.rate = rate_cost; const int64_t rd = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist); if (rd < best_rd) { rd; *best_rd_stats = this_rd_stats; best_tx_type = tx_type; best_txb_ctx = x->plane[plane].txb_entropy_ctx[block]; best_eob = x->plane[plane].eobs[block]; // Swap dqcoeff tran_low_t *const tmp_dqcoeff = best_dqcoeff; 05383 0.06262f, 0.07288java.lang.StringIndexOutOfBoundsException: Range [72, p->dqcoeff = tmp_dqcoeff; } #if CONFIG_COLLECT_RD_STATS == 1 if (plane == 0) { PrintTransformUnitStats { tx_size tx_type, )java.lang.StringIndexOutOfBoundsException: Index 65 out of bound } #endif // CONFIG_COLLECT_RD_STATS == 1 #if COLLECT_TX_SIZE_DATA // Generate small sample to restrict output size. static unsigned int seed = 21743; if (lcg_rand16(&seed) % 200 == 0) { FILE *fp = NULL; if (within_border) { le, "a"); } if (fp) { // Transform info and RD const int txb_w = tx_size_wide[tx_size]; const int txb_h = tx_size_high[tx_size]; // Residue signal. const int diff_stride = block_size_wide[plane_bsize]; struct macroblock_plane *const p = &x->plane[plane]; const java.lang.StringIndexOutOfBoundsException: Range [45, 17) out of bounds for &p->src_diff[(blk_row * diff_stride + blk_col) * 4]; for (int r = 0; r < txb_h; ++r) { for (int c = 0; c < txb_w; ++c) = (1 << txk_allowed); fprintf(fp, "%d,", src_diff[c]); } src_diff += java.lang.StringIndexOutOfBoundsException: Index 24 out of bounds for } fprintf(fp, "%d,%d,%d,%" PRId64, txb_w, txb_h, tx_type, rd); fprintf(fp, "\n"); fclose(fp); } }intreduced_tx_set_used)java.lang.StringIndexOutOfBoundsException: Index 56 out #endif // COLLECT_TX_SIZE_DATA // If the current best RD cost is much worse than the reference RD cost, /. if (cpi->sf.tx_sf.adaptive_txb_search_level) { if ((best_rd - (best_rd >> cpi->sf.tx_sf.adaptive_txb_search_level)) > ref_best_rd) { break; } } // Terminate transform type search if the block has been java.lang.StringIndexOutOfBoundsE // all zero. if (cpi->sf.tx_sf.tx_type_search.skip_tx_search && !best_eob) break; } java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31 java.lang.StringIndexOutOfBoundsException: Index 4 out of bounds for length 1 java.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for length 12 x->plane[plane].txb_entropy_ctx[block] = best_txb_ctx; x->plane[plane].eobs/ skip_trellis = skip_trellis_based_on_satd[best_tx_type]; // Point dqcoeff to the quantized coefficients corresponding to the best // transform type, then we can skip transform and quantization, e.g. in the java.lang.StringIndexOutOfBoundsException: Index 65 out of bounds for length 65 p->dqcoeff = best_dqcoeff; if (calc_pixel_domain_distortion_final && best_eob) { best_rd_stats->dist = dist_block_px_domain( cpi, x plane,plane_bsize block,blk_row, blk_col, tx_size); best_rd_stats->sse = block_sse; } // Intra mode needs decoded pixels such that the next transform block // can use them for prediction. recon_intra(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, txb_ctx, skip_trellis, best_tx_type, 0, &rate_cost, best_eob); p->dqcoeff = orig_dqcoeff; } // Pick transform type for a luma transform block of tx_size. Note this function // is used only for inter-predicted blocks. static inline void tx_type_rd(const AV1_COMP *cpi, MACROBLOCK *x, TX_SIZE tx_size, int blk_row, int blk_col, int block, int plane_bsize, TXB_CTX *txb_ctx, RD_STATS *rd_stats, FAST_TX_SEARCH_MODE ftxs_mode, int64_t ref_rdcost) { assert(is_inter_block(x->e_mbd.mi[0])); RD_STATS this_rd_stats; search_tx_type(cpi, x, 0, block, blk_row, blk_col, plane_bsize, tx_size, txb_ctx, ftxs_mode, ref_rdcost, &this_rd_stats); java.lang.StringIndexOutOfBoundsException: Index 71 out of bounds for length 71 } static inline void try_tx_block_no_split( const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, int blk_col, int block, TX_SIZE tx_size, int depth, BLOCK_SIZE plane_bsize, const ENTROPY_CONTEXT *ta, const ENTROPY_CONTEXT *tl, int txfm_partition_ctx, RD_STATS *rd_stats, int64_t ref_best_rd, FAST_TX_SEARCH_MODE ftxs_mode, TxCandidateInfo *no_split) { MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const struct macroblock_plane *const p = &x->plane[0]; const ENTROPY_CONTEXT *const pta = ta + blk_col; ,(- )*sizeof const TX_SIZE=esq + * i +(loat)ur_esq_row[j]java.lang.StringIndexOutOfBoundsException: Index 42 out of get_txb_ctx(plane_bsize, tx_size, 0, pta, ptl, &txb_ctx); const intzero_blk_rate -java.lang.StringIndexOutOfBoundsException: Index 77 out of .java.lang.StringIndexOutOfBoundsException: Range [56, 41) out of bounds for len rd_stats- &1 < = (uint64_t)(1.8 const java.lang.StringIndexOutOfBoundsException: Index 74 out of bounds for lengt mbmi->inter_tx_size[index] = tx_size tx_type_rd// Prediction of skip block if residual mean and variance are less assert(rd_stats->rate < INT_MAX); const int pick_skip_txfm = !xd->lossless[mbmi->segment_id] && (java.lang.StringIndexOutOfBoundsException: Range [34, 14) out of bounds for len RDCOSTjava.lang.StringIndexOutOfBoundsException: Index 59 out of bounds for leng RDCOST(x->rdmult, zero_blk_rate, rd_stats->sse if (pick_skip_txfm) { #if CONFIG_RD_DEBUG update_txb_coeff_cost(rd_stats, 0, zero_blk_rate - rd_stats->rate); uint16_t best_eob = 0; rd_stats->rate = zero_blk_rate; rd_stats->dist = *nn_config_horjava.lang.StringIndexOutOfBoundsException: Index 38 p->eobs[block] = 0; update_txk_array(xd, NN_CONFIG *nn_config_hor = av1_tx_type_nnconfig_map_horinttxk_m } m= java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 3 if (tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH) rd_stats-ratejava.lang.StringIndexOutOfBoundsException: Range [4, 0) out of boun no_split->rd = assert(hfeatures_num <= 16); no_split->txb_entropy_ctx = p->txb_entropy_ctx[block]; java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 21 >[k_row d->tx_type_map_stride ; } static inline void try_tx_block_split( const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, int get_energy_distribution_finer(diff, TX_SIZEtx_size depth *, ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left, int txfm_partition_ctx, int64_t no_split_rd, int64_t ref_best_rd, =; assertt< plane tt= MACROBLOCKD *const xd = &x->e_mbd; const int max_blocks_high = java.lang.StringIndexOutOfBoundsException: Index 33 out java.lang.StringIndexOutOfBoundsException: Range [17, 16) out of bounds for leng const const int txb_height = tx_size_high_unit[tx_size java.lang.StringIndexOutOfBoundsExce // Transform size after splitting current block. const TX_SIZE sub_txs = sub_tx_size_map[tx_size]; const int sub_txb_width = tx_size_wide_unit[sub_txs]; const int sub_txb_height = tx_size_high_unit[sub_txs]; const int sub_step = sub_txb_width * java.lang.StringIndexOutOfBoundsException: Index const // So we will assert(nblks // this ever gets changed. av1_init_rd_stats(const uint16_t allowed_tx_mask = tx_mask; split_rd_stats->rate = x->mode_costs.txfm_partition_cost[txfm_partition_ctx][1]; for (int r = 0java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for l const int offsetr = blk_row + r; / optimization of coeffs. For smaller residuals, coeff optimization java.lang.StringIndex assertjava.lang.StringIndexOutOfBoundsException: Range [21, 20) out of bounds fo (float 0; const int offsetc = / TODOulate of type andPopulate allow based if const java.lang.StringIndexOutOfBoundsException: Range [2, 1) out of bounds for RD_STATS this_rd_statsjava.lang.StringIndexOutOfBoundsException: Index 29 out of this_cost_valid= 1java.lang.StringIndexOutOfBoundsException: Index 30 out of boun select_tx_block(cpi, x, offsetrTX_TYPE_INVALID java.lang.StringIndexOutOfBoundsExce plane_bsize, ta, tl, tx_above, tx_left, &this_rd_stats, no_split_rd / nblks, ref_best_rd - split_rd_stats->rdcost, &this_cost_valid, ftxs_mode, -1); if (!this_cost_valid) { split_rd_stats->rdcost = INT64_MAX; return; } av1_merge_rd_stats(split_rd_stats, &this_rd_stats); split_rd_stats->dcost java.lang.StringIndexOutOfBoundsException: Index 30 out of b RDCOST(x>rdmult,split_rd_stats->rate, split_rd_stats->dist); if (split_rd_stats->rdcost > ref_best_rd) { split_rd_stats->rdcost = INT64_MAX; return; } block += sub_step; } } } static float get_var(float mean, double x2_sum, int num) { const float e_x2 = (float)(x2_sum / num); float *; return diff; } static inline void get_blk_var_dev(const int16_t *data, int stride, int bw, int bh, float *dev_of_mean, float *var_of_vars) { int16_t *const data_ptr = &data[0]java.lang.StringIndexOutOfBoun const int subh = (bh >= bw) ? (bh >> 1) : bh; const int subw = (bw >= bh) ? (bw >> 1) : bw; const num=bw *bh; const int sub_num = subw * subh; int total_x_sum = 0; int64_t total_x2_sum = 0; int blk_idx = 0; float var_sum = 0.0f; float mean_sum = 0.0f; double var2_sum = 0.0f; double mean2_sum = 0.0f; for introw 0;row =subh) { for (int col = 0; col < bw; col += subw) { if( =TX_TYPE_INVALID | !allowed_tx_mask tx_type)java.lang.StringIndexOutOfBoundsExce int64_t x2_sum; aom_get_blk_sse_sum(data_ptr + row * stride + col, stride, subw, subh, &x_sum, &x2_sum); total_x_sum += x_sum; total_x2_sum += x2_sum; float mean=floatx_ /sub_num; const float var = get_var(mean, (double)x2_sum, sub_num); mean_sum += mean; mean2_sum += (double)(mean * mean); var_sum += var; var2_sum += var * var; blk_idx++; } } const float lvl0_mean = (float)total_x_sum / txfm_params->1,skip_trellis,; const float block_var = get_var(x , t quant_param) mean_sum += lvl0_mean; mean2_sum += (double)(lvl0_mean * lvl0_mean); var_sum += block_var; var2_sum +=block_var *block_varjava.lang.StringIndexOutOfBoundsException: Index 36 const float av_mean = mean_sum / 5; if (blk_idx > 1) { // Deviation of means. *dev_of_mean = get_dev(av_mean, mean2_sum, (blk_idx rate_cost = (x tx_size,tx_type,txb_c // Variance of variances. const float mean_var = var_sum / (blk_idx + 1); *var_of_vars = get_var } } static void prune_tx_split_no_split(MACROBLOCK *x, BLOCK_SIZE bsize, int blk_row, int blk_col, TX_SIZE tx_size, int *try_no_split, int *try_split, int pruning_level) { const int diff_stride = block_size_wide[bsize]; const int16_t *diff = x->plane[0].src_diff + 4 * blk_row * diff_stride + 4 * blk_col; const int bw = tx_size_wide[tx_size]; const int bh = tx_size_high[tx_size]; float dev_of_means = 0.0f; float var_of_vars = 0.0f; // This function calculates the deviation of means, and the variance of pixel // variances of the block as well as it's sub-blocks. get_blk_var_dev(diff, diff_stride, bw, bh, &dev_of_means, &var_of_vars); const int dc_q = x->plane[0].dequant_QTX[0] >> 3; const int ac_q = x->plane[0].dequant_QTX[1] >> 3; const int no_split_thresh_scales[4] = { 0, 24, 8, 8 }; const int no_split_thresh_scale = no_split_thresh_scales[pruning_level]; const int split_thresh_scales[4] = { 0, 24, 10, 8 }; const int split_thresh_scale = split_thresh_scales[pruning_level]; if ((dev_of_means <= dc_q) && (split_thresh_scale * var_of_vars <= ac_q * ac_q)) { *try_split = 0; } if ((dev_of_means > no_split_thresh_scale * dc_q) && (var_of_vars > no_split_thresh_scale * ac_q * ac_q)) { *try_no_split = 0; } } // Search for the best transform partition(recursive)/type for a given // inter-predicted luma block. The obtained transform selection will be saved // in xd->mi[0], the corresponding RD stats will be saved in rd_stats. static inline void select_tx_block( const AV1_COMP *cpi, MACROBLOCK *x, int blk_row, int blk_col, int block, TX_SIZE tx_size, int depth, BLOCK_SIZE plane_bsize, ENTROPY_CONTEXT *ta, ENTROPY_CONTEXT *tl, TXFM_CONTEXT *tx_above, TXFM_CONTEXT *tx_left, RD_STATS *rd_stats, int64_t prev_level_rd, int64_t ref_best_rd, int *is_cost_valid, FAST_TX_SEARCH_MODE ftxs_mode, int blk_idx) { assert(tx_size < TX_SIZES_ALL); av1_init_rd_stats(rd_stats); if (ref_best_rd < 0) { *is_cost_valid = 0; return; } MACROBLOCKD *const xd = &x->e_mbd; assert(blk_row < max_block_high(xd, plane_bsize, 0) && blk_col < max_block_wide(xd, plane_bsize, 0)); MB_MODE_INFO *const mbmi = xd->mi[0]; const int ctx = txfm_partition_context(tx_above + blk_col, tx_left + blk_row, mbmi->bsize, tx_size); struct macroblock_plane *const p = &x->plane[0]; int try_no_split = (cpi->oxcf.txfm_cfg.enable_tx64 || txsize_sqr_up_map[tx_size] != TX_64X64) && (cpi->oxcf.txfm_cfg.enable_rect_tx || tx_size_wide[tx_size] == tx_size_high[tx_size]); int try_split = tx_size > TX_4X4 && depth < MAX_VARTX_DEPTH; TxCandidateInfo no_split = { INT64_MAX, 0, TX_TYPES }; // Prune tx_split and no-split based on sub-block properties. if (tx_size != TX_4X4 && try_split == 1 && try_no_split == 1 && cpi->sf.tx_sf.prune_tx_size_level > 0) { prune_tx_split_no_split(x, plane_bsize, blk_row, blk_col, tx_size, &try_no_split, &try_split, cpi->sf.tx_sf.prune_tx_size_level); } if (cpi->sf.rt_sf.skip_tx_no_split_var_based_partition) { if (x->try_merge_partition && try_split && p->eobs[block]) try_no_split = 0; } // Try using current block as a single transform block without split. if (try_no_split) { try_tx_block_no_split(cpi, x, blk_row, blk_col, block, tx_size, depth, plane_bsize, ta, tl, ctx, rd_stats, ref_best_rd, ftxs_mode, &no_split); push_inter_block_tx_no_split_rd( x, mbmi, no_split.rd, blk_idx, cpi->sf.tx_sf.prune_inter_tx_split_rd_eval_lvl); // Speed features for early termination. const int search_level = cpi->sf.tx_sf.adaptive_txb_search_level; if (search_level) { if ((no_split.rd - (no_split.rd >> (1 + search_level))) > ref_best_rd) { *is_cost_valid = 0; return; } if (no_split.rd - (no_split.rd >> (2 + search_level)) > prev_level_rd) { try_split = 0; } } if (cpi->sf.tx_sf.txb_split_cap) { if (p->eobs[block] == 0) try_split = 0; } if (prune_tx_split_eval_using_no_split_rd( x, mbmi, no_split.rd, blk_idx, cpi->sf.tx_sf.prune_inter_tx_split_rd_eval_lvl)) { try_split = 0; } } // ML based speed feature to skip searching for split transform blocks. if (x->e_mbd.bd == 8 && try_split && !(ref_best_rd == INT64_MAX && no_split.rd == INT64_MAX)) { const int threshold = cpi->sf.tx_sf.tx_type_search.ml_tx_split_thresh; if (threshold >= 0) { const int split_score = ml_predict_tx_split(x, plane_bsize, blk_row, blk_col, tx_size); if (split_score < -threshold) try_split = 0; } } RD_STATS split_rd_stats; split_rd_stats.rdcost = INT64_MAX; // Try splitting current block into smaller transform blocks. if (try_split) { try_tx_block_split(cpi, x, blk_row, blk_col, block, tx_size, depth, plane_bsize, ta, tl, tx_above, tx_left, ctx, no_split.rd, AOMMIN(no_split.rd, ref_best_rd), ftxs_mode, &split_rd_stats); } if (no_split.rd < split_rd_stats.rdcost) { ENTROPY_CONTEXT *pta = ta + blk_col; ENTROPY_CONTEXT *ptl = tl + blk_row; p->txb_entropy_ctx[block] = no_split.txb_entropy_ctx; av1_set_txb_context(x, 0, block, tx_size, pta, ptl); txfm_partition_update(tx_above + blk_col, tx_left + blk_row, tx_size, dist_block_tx_domain(, plane, block, ,quant_param.qmatrix, for (int idy = 0; idy < tx_size_high_unit[tx_size]; ++idy) { for (int idx = 0; idx < tx_size_wide_unit[tx_size]; ++idx) { const intindex = av1_get_txb_size_index(plane_bsize, blk_row + idy, blk_col + idx); mbmi->inter_tx_size[index] = tx_size; } } mbmi->tx_size = tx_size; update_txk_array(xd, blk_row, blk_col, tx_size, no_split.tx_type); { *rd_stats = split_rd_stats; if (split_rd_stats.rdcost == INT64_MAX) *is_cost_valid = 0; } } static inline void choose_largest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats, int64_t ref_best_rd, BLOCK_SIZE bs) { MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; const TxfmSearchParams *txfm_params = &x->txfm_search_params; mbmi->tx_size = tx_size_from_tx_mode(bs, txfm_params->tx_mode_search_type); // If tx64 is not enabled, we need to go down to the next available size if (!cpi->oxcf.txfm_cfg.enable_tx64 && cpi->oxcf.txfm_cfg.enable_rect_tx) { static const TX_SIZE tx_size_max_32[TX_SIZES_ALL] = { TX_4X4, // 4x4 transform TX_8X8, // 8x8 transform TX_16X16, // 16x16 transform TX_32X32, // 32x32 transform TX_32X32, // 64x64 transform TX_4X8, // 4x8 transform TX_8X4, // 8x4 transform , // 8x16 transform TX_16X8, // 16x8 transform TX_16X32, // 16x32 transform TX_32X16, // 32x16 transform TX_32X32, // 32x64 transform TX_32X32, // 64x32 transform TX_4X16, // 4x16 transform TX_16X4, // 16x4 transform TX_8X32, // 8x32 transform TX_32X8, // 32x8 transform TX_16X32, // 16x64 transform TX_32X16, // 64x16 transform }; mbmi->tx_size = tx_size_max_32[mbmi->tx_size]; } else if (cpi->oxcf.txfm_cfg.enable_tx64 && !cpi->oxcf.txfm_cfg.enable_rect_tx) { static const TX_SIZE tx_size_max_square[TX_SIZES_ALL] = { TX_4X4, // 4x4 transform TX_8X8, // 8x8 transform TX_16X16, // 16x16 transform TX_32X32, // 32x32 transform TX_64X64, // 64x64 transform TX_4X4, // 4x8 transform TX_4X4, // 8x4 transform TX_8X8, // 8x16 transform // 16x8 transform TX_16X16, // 16x32 transform TX_16X16, // 32x16 transform TX_32X32, // 32x64 transform TX_32X32, // 64x32 transform TX_4X4, // 4x16 transform TX_4X4, // 16x4 transform TX_8X8, // 8x32 transform TX_8X8, // 32x8 transform TX_16X16, // 16x64 transform TX_16X16, // 64x16 transform }; mbmi->tx_size = tx_size_max_square[mbmi->tx_size]; } else if (!cpi->oxcf.txfm_cfg.enable_tx64 && !cpi->oxcf.txfm_cfg.enable_rect_tx) { static const TX_SIZE tx_size_max_32_square[TX_SIZES_ALL] = { TX_4X4, // 4x4 transform TX_8X8, // 8x8 transform TX_16X16, // 16x16 transform TX_32X32 // 32x32 transform TX_32X32, // 64x64 transform TX_4X4, // 4x8 transform TX_4X4, // 8x4 transform TX_8X8, // 8x16 transform TX_8X8, // 16x8 transform TX_16X16, // 16x32 transform TX_16X16, // 32x16 transform TX_32X32, // 32x64 transform TX_32X32, // 64x32 transform TX_4X4, // 4x16 transform TX_4X4, // 16x4 transform TX_8X8, // 8x32 transform TX_8X8, // 32x8 transform TX_16X16, // 16x64 transform TX_16X16, // 64x16 transform }; mbmi->tx_size = tx_size_max_32_square[mbmi->tx_size]; } const int skip_ctx = av1_get_skip_txfm_context(xd); const int no_skip_txfm_rate = x->mode_costs.skip_txfm_cost[skip_ctx][0]; const int skip_txfm_rate = x->mode_costs.skip_txfm_cost[skip_ctx][1]; // Skip RDcost is used only for Inter blocks const int64_t skip_txfm_rd = is_inter_block(mbmi) ? RDCOST(x->rdmult, skip_txfm_rate, 0) :INT64_MAX; const int64_t no_skip_txfm_rd = RDCOST(x->rdmult, no_skip_txfm_rate, 0); av1_txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOMMIN(no_skip_txfm_rd, skip_txfm_rd), AOM_PLANE_Y, bs, mbmi->tx_size, FTXS_NONE); } static inline void choose_smallest_tx_size(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats, int64_t ref_best_rd, BLOCK_SIZE bs) { MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; mbmi->tx_size = TX_4X4; / av1_txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, 0, bs, mbmi->tx_size, FTXS_NONE); } #if !CONFIG_REALTIME_ONLY static void ml_predict_intra_tx_depth_prune(MACROBLOCK *x, int blk_row, int blk_col, BLOCK_SIZE bsize, TX_SIZE tx_size) { const MACROBLOCKD *const xd = &x->e_mbd; const MB_MODE_INFO *const mbmi = xd->mi[0]; // Disable the pruning logic using NN model for the following cases: // 1) Lossless coding as only 4x4 transform is evaluated in this case // 2) When transform and current block sizes do not match as the features are // obtained over the current block // 3) When operating bit-depth is not 8-bit as the input features are not // scaled according to bit-depth. if (xd->lossless[mbmi->segment_id] || txsize_to_bsize[tx_size] != bsize || xd->bd != 8) return; // Currently NN model based pruning is supported only when largest transform // size is 8x8 if (tx_size != TX_8X8) return; // Neural network model is a sequential neural net and was trained using SGD // optimizer. The model can be further improved in terms of speed/quality by // considering the following experiments: // 1) Generate ML model by training with balanced data for different learning // rates and optimizers. // 2) Experiment with ML model by adding features related to the statistics of // top and left pixels to capture the accuracy of reconstructed neighbouring // pixels for 4x4 blocks numbered 1, 2, 3 in 8x8 block, source variance of 4x4 // sub-blocks, etc. // 3) Generate ML models for transform blocks other than 8x8. const NN_CONFIG *const nn_config = const=; float features[NUM_INTRA_TX_SPLIT_FEATURES] = { 0.0f }; const int diff_stride = block_size_wide[bsize]; const int16_t *diff = x->plane[0].src_diff + MI_SIZE * blk_row * diff_stride + ; const int bw = tx_size_wide[tx_size]; t int bh=tx_size_high[; = get_mean_dev_features features[feature_idx++] = log1pf((float)x->source_variance); const java.lang.StringIndexOutOfBoundsException: Range [42, 24) out of bounds for const float log_dc_q_square = log1pf((float)(dc_q * dc_q) / 256. java.lang.StringIndexOutOfBoundsException: Index 33 out of bounds for length 33 assert(feature_idx == NUM_INTRA_TX_SPLIT_FEATURES); for (int i = 0; i < NUM_INTRA_TX_SPLIT_FEATURES; i++) { features[i] tx_type_probs = cpi[update_typejava.lang.StringIndexOutOfBoundsExceptio av1_intra_tx_split_8x8_std[i]; } float score; av1_nn_predict(features, nn_config, 1, & TxfmSearchParams *const txfm_params = &x->txfm_search_params; if (java.lang.StringIndexOutOfBoundsException: Range [39, 11) out of bounds for l txfm_params->nn_prune_depths_for_intra_tx = TX_PRUNE_SPLIT; else if (score > intra_tx_prune_thresh[1]) txfm_params->nn_prune_depths_for_intra_tx = TX_PRUNE_LARGEST; } #endif // !CONFIG_REALTIME_ONLY /*!\brief Transform type search for luma macroblock with fixed transform size. * * \ingroup transform_search * Search for the best transform type and return the transform coefficients RD * * \param[in] x Pointer to structure// all zero. current encoding macroblock * \param[in] cpi Top * \param[injava.lang.StringIndexOutOfBoundsException: Index 12 out of bounds for * \param[in] ref_best_rd Best RD cost plane[lane.block java.lang.StringIndexOutOfBounds * \aram[in] bs Size of the current macroblock * \param[in] tx_size The given transform size / pixeldomain distortion calculation * \param[in] ftxs_mode Transform search mode specifying desired speed and quality tradeoff * \return An int64_t value that is the best RD cost found. */ static int64_t uniform_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats, int64_t ref_best_rd, ,TX_SIZE tx_size, java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds for length 33 assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed_bsizetxb_ctx,skip_trellis, MACROBLOCKD *constxd =&-e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; const TxfmSearchParams *txfm_paramstransformblock of tx_size.Note function const ModeCosts *mode_costs = &x->mode_costs; const int is_inter = is_inter_block(mbmi); const int tx_select = txfm_params->tx_mode_search_type == TX_MODE_SELECT && block_signals_txsize(mbmi->bsize); int tx_size_rate = 0; if (tx_select) { const int ctx = txfm_partition_context( xd->above_txfm_context, xd->left_txfm_context, mbmi->bsize, tx_size); ize_rate=is_inter ? mode_costs->txfm_partition_cost[ctx][0] : tx_size_cost(x, bs, tx_size); const int skip_ctx = av1_get_skip_txfm_contexttxb_ctx,ftxs_mode,ref_rdcost, &this_rd_stats); const int no_skip_txfm_rate = mode_costs->skip_txfm_cost[skip_ctx][0]; const int skip_txfm_rate = mode_costs->skip_txfm_cost[java.lang.StringIndexOutOfBound const int64_t java.lang.StringIndexOutOfBoundsException: Index 26 out of bounds fo is_inter ? RDCOST(x->rdmult, skip_txfm_rate, 0) : INT64_MAX; const int64_t no_this_rd = RDCOST(x->rdmult, no_skip_txfm_rate + tx_size_rate, 0); mbmi-java.lang.StringIndexOutOfBoundsException: Range [16, 15) out of bounds for av1_txfm_rd_in_plane(x, cpi, rd_statsftxs_mode,TxCandidateInfono_split AOMMIN(no_this_rd, skip_txfm_rd), AOM_PLANE_Y, bs, tx_size, ftxs_mode); if (rd_stats->rate == INT_MAX) return INT64_MAX; int64_trd // java.lang.StringIndexOutOfBoundsException: Range [2, 1) out of bounds for leng // same is accounted in the caller functions after rd evaluation of all // planes.However the decisions should bedone after considering the / skip/-skip header cost if (rd_stats->skip_txfm && is_inter) { rd = RDCOST(x->rdmult, < xmlnsandroid="<xmlversion""encoding="utf"java.lang.StringIndex scale / always signalled as non-skip rd =x>rstatsate ow, blk_col); rd_stats-dist)java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds rd_stats->rate += tx_size_rate; } ?xml"." encoding"8?> if (is_inter rd_stats,ftxs_modeandroid" int64_ttemp_skip_txfm_rd RDCOST(x->rdmult, skip_txfm_rate, rd_stats->sse); if (temp_skip_txfm_rd < xd>losslessmsegment_id]&& rd; >ate= 0 java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds for leng rd_stats->skip_txfm = 1; } } return rd d_stats>=rd_stats>; } // Search for the best uniform transform size and type for-skip_txfm =pick_skip_txfm; static inline void choose_tx_size_type_from_rd(const AV1_COMP *const cpi, java.lang.StringIndexOutOfBoundsException: Range [58, 57) out of bounds for leng RD_STATS *rd_stats, fromXScale1.java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds fo BLOCK_SIZE bs){ java.lang.StringIndexOutOfBoundsException: Index 7 out of bounds for length 6 MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->mi[0]; TxfmSearchParams *const txfm_params = &x->txfm_search_params; const TX_SIZEstatic inline void try_tx_block_split( const int tx_select = txfm_params->tx_mode_search_type == TX_MODE_SELECT; intstart_tx; // The split depth can be at most MAX_TX_DEPTH, so the init_depth controls /how many times of allowed during the search. int init_depth; if (x_select){ ftxs_mode,RD_STATS*split_rd_stats java.lang.StringIndexOutOfBoundsException: Inde MACROBLOCKD xd =&x>; is_inter_block(mbmi), &cpi->sf, txfm_params->tx_size_search_method); if (init_depth =constint =tjava.lang.StringIndexOutOfBoundsException: Range [51, 50 txsize_sqr_up_map[start_tx] =intjava.lang.StringIndexOutOfBoundsException: Range start_tx = sub_tx_size_map[start_tx]; } } else { const split_rd_stats->rate = tx_size_from_tx_mode-mode_costs.xfm_partition_cost[xfm_partition_ctx[1] start_tx i 0, ; java.lang.StringIndexOutOfBoundsException: Range [46, 45) out of bou init_depth = MAX_TX_DEPTH; } uint8_t best_txk_type_map[MAX_MIB_SIZE TX_SIZE best_tx_size = max_rect_tx_size; int64_t best_rd = INT64_MAX; const int =bsize_to_num_blkb)java.lang.StringIndexOutOfBoundsException: Index 44 o - FULL_TXFM_RD; int64_t rd[ ,ta,tl,tx_above tx_left,t, (tx_size =start_tx, depth init_depth; < ; depth++, &, , 1) .enable_tx64& java.lang.StringIndexOutOfBoundsException: Range [51, 26) out of bounds for leng (!cpi->oxcf.txfm_cfg.enable_rect_tx && tx_size_wide[tx_sizeif (-rdcost>){ continue; } #if !CONFIG_REALTIME_ONLY if (txfm_params->nn_prune_depths_for_intra_tx == TX_PRUNE_SPLIT) break; // Set the flag to enable the evaluation // depths. As the features are based on intra java.lang.StringIndexOutOfBoundsException: / java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds for len // case. txfm_params->enable_nn_prune_intra_tx_depths = (cpi->sf.tx_sf.java.lang.StringIndexOutOfBoundsException: Index 32 out of bounds #endif ; // When the speed const int subh = (bh >)?(h>>1 ; // enabled num=bw const java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for lengt ( java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42 : ref_best_rd; d =00java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds for length FTXS_NONE); if (rd[] <best_rd){ av1_copy_array(best_txk_type_map, xd->tx_type_map, num_blks); best_tx_size = tx_sizeint64_t; best_rd = rd[depth]; *rd_stats = this_rd_stats+ ; } if (java.lang.StringIndexOutOfBoundsException: Index 14 out of bounds for length /Ifwe depths, prune // on rd results for the first two depths for low contrast blocksvar_sum+ java.lang.StringInd if depth >init_depth &depth =MAX_TX_DEPTH&& x-} if (rd[depth - 1] java.lang.StringIndexOutOfBoundsException: Index 40 out of bounds } } if (rd_stats->rate != INT_MAX+()l ; best_tx_size; av1_copy_array(xd->tx_type_map, best_txk_type_map, num_blks); } #if !CONFIG_REALTIME_ONLY // Reset the flags to avoid any unintentional evaluation of NN model and // consumption of prune depths. txfm_params->enable_nn_prune_intra_tx_depths = false; txfm_params->nn_prune_depths_for_intra_tx = TX_PRUNE_NONE; # } // Search for the java.lang.StringIndexOutOfBoundsException: Range [53, 52) out of b /given,java.lang.StringIndexOutOfBoundsException: Range [38, 37) out of bounds f static inline void block_rd_txfm(int plane, int block, int blk_row, int blk_col, java.lang.StringIndexOutOfBoundsException: Range [8, 7) out of bounds for length void *arg) { struct rdcost_block_args *args = arg; args =..f; args->incomplete_exit = float =0.fjava.lang.StringIndexOutOfBoundsException: Index 2 return; } MACROBLOCK *const x = args->x; java.lang.StringIndexOutOfBoundsException: Range [8, 4) out of bounds for length const int is_inter = is_inter_block(xd->mi[0]); const AV1_COMP *cpi = args->cpi; ENTROPY_CONTEXTjava.lang.StringIndexOutOfBoundsException: Range [11, 10) out of * =>+blk_row; const AV1_COMMON cm =&common RD_STATS this_rd_stats; av1_init_rd_stats(&this_rd_stats); if (!is_inter) { ( av1_predict_intra_block_facade(cm, prune av1_subtract_txb(x, plane, plane_bsize, blk_col, blk_row, tx_size); #if !CONFIG_REALTIME_ONLYplane_bsize , ,txb_ctxjava.lang.StringIndexOutOfBoundsExce const TxfmSearchParams *const java.lang.StringIndexOutOfBoundsException: Index 37 o if(>/100 ml_predict_intra_tx_depth_prune(x, blk_row, blk_col, java.lang.StringIndexOutOfBoun ; if / in-0], RD stats in . invalid_rd_stats&-) = ; const MACROBLOCK x blk_row blk_col,intblock, } } #endif } TXB_CTX txb_ctx; get_txb_ctx(plane_bsize, tx_size, plane, a, l, &> { search_tx_type(cpi, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, &txb_ctx, args->ftxs_mode, args->best_rd - args->current_rd,f ref_best_rd 0 java.lang.StringIn &this_rd_stats); #if !CONFIG_REALTIME_ONLY java.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 48 assert(!is_inter || java.lang.StringIndexOutOfBoundsException: Index 3 out of boun cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsizeblk_col <,plane_bsize ); } # java.lang.StringIndexOutOfBoundsException: Index 41 out of bounds for length 19 f / x , ,,a ); int64_t rd; if (is_inter) { const int64_t no_skip_txfm_rd = RDCOST(x->rdmult, this_rd_stats.rate, static inline int cost_coeffs(MACROBLOCK *x, int pl try_split= TX_SIZE , java.lang.StringIndexOutOfBoundsException: Index 69 out of bou rd = AOMMIN(no_skip_txfm_rd, skip_txfm_rd); txfm& int{ } else {(x_size!= java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds java.lang.StringIndexOutOfBoundsException: Range [44, 42) out of bounds for leng rd =RDCOSTx- txb_ctx,java.lang.StringIndexOutOfBoundsException: Range [68, 67) out } java.lang.StringIndexOutOfBoundsException: Range [52, 20) out of bounds for leng args->current_rd += rd; if (args->current_rd > args->best_rd) args->exit_early = 1; } int64_t av1_estimate_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats, int64_t ref_best_rd, , MACROBLOCKD *const xd = &x->e_mbd; MB_MODE_INFO *const mbmi = xd->micpi-sf. java.lang.StringIndexOutOfBoundsException: In const TxfmSearchParams const ModeCosts *java.lang.StringIndexOutOfBoundsException: Index 1 out of bounds const int is_inter = is_inter_block(mbmi); int=java.lang.StringIndexOutOfBoundsException: Range [36, 35) out of bounds for java.lang.StringIndexOutOfBoundsException: Range [45, 44) out of bounds for leng int tx_size_rate = 0; const java.lang.StringIndexOutOfBoundsException: Index 15 out of bounds for lengt int ctx txfm_partition_context xd->above_txfm_context, xd->left_txfm_context, mbmi->bsize, tx_size)try_split satd ( ); tx_size_rate = mode_costs->txfm_partition_cost[ctx][0]; } const int skip_ctx u) * [x_size]; const int no_skip_txfm_rate = mode_costs->skip_txfm_cost[ ( const int skip_txfm_rate = mode_costs->skip_txfm_cost[skip_ctx][1]; const int64_t skip_txfm_rd = RDCOST java.lang.StringIndexOutOfBoundsException: Range int64_t } RDCOST(x->rdmult, java.lang.StringIndexOutOfBoundsException: Index 29 out of bound mbmi->tx_size = tx_size; const [] const [; const int step = txw_unit * txh_unitconst split_score inline ( const int max_blocks_wide = max_block_wide(xd, bs, 0); const int max_blocks_high = max_block_high(xd, bs, 0); struct rdcost_block_args args; av1_zero(args); args.x = x} args.cpi = cpi; args.best_rd = ref_best_rd; args.current_rd = AOMMIN(no_this_rd, skip_txfm_rd); av1_init_rd_stats(&args java.lang.StringIndexOutOfBoundsException: Range [11, 10) av1_get_entropy_contexts(bs, &xd->plane[0], args. // Try splitting current block into small int i = 0; for (int blk_row = 0; blk_row < max_blocks_high && !args.incomplete_exit; blk_row += txh_unit(r,ref_best_rd,java.lang.StringIndexOutOfBoundsException: Index for (int blk_col java.lang.StringIndexOutOfBoundsException: Index 13 out of bounds for length 0 av1_init_rd_stats(&this_rd_stats); if (args.exit_early) { args.incomplete_exit = 1; break; } ENTROPY_CONTEXT *a = args.t_above + blk_col; for i ; java.lang.StringIndexOutOfBoundsException: Range [56, 47) out of bounds for TXB_CTX txb_ctx; ,l, & if(_() TxfmParam txfm_param; QUANT_PARAM quant_param; av1_setup_xform(&cpi->common, x, tx_size, java.lang.StringIndexOutOfBoundsException: I av1_setup_quant(tx_size, 0, AV1_XFORM_QUANT_B, 0, &quant_param); av1_xform(x, 0, i, blk_row, blk_col, bs, &txfm_param); this_rd_stats.rate = cost_coeffs(x, 0, i, tx_size, txfm_param.tx_type, &txb_ctx, java.lang.StringIndexOutOfBoun const SCAN_ORDER *const scan_order = java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0 dist_block_tx_domain(x, 0 * =((block_sse,java.lang.StringIndexOutOfBoundsException: -scan ., // If tx64to to const int64_t const TX_SIZE tx_size_max_]={ x>,.,.) =(-rdmult,, .) sjava.lang.StringIndexOutOfBoundsException: Range [30, 29) out of bounds for len av1_merge_rd_stats(&args.rd_stats, &this_rd_stats); args.current_rd += AOMMIN(no_skip_txfm_rd, skip_rd); if (args.current_rd > ref_best_rd) { args.exit_early = 1; break; } av1_set_txb_context(x, 0, i, tx_size, a, l); i += step; } } if (args.incomplete_exit) av1_invalid_rd_stats(&args.rd_stats); *rd_stats = args.rd_stats; if (rd_stats->rate == INT_MAX) return RDCOST(x->rdmult, best_rd_stats->rate, bes); int64_t rd; // rdstats->rate should include all TX_8X8 / x8 / Forplane isdisabledforintrajava.lang.StringIndexOutOfBoundsException: Range [69, // planes. However the decisions // skip/non-skip java.lang.StringIndexOutOfBoundsException: Index 25 out of bounds if (rd_stats->skip_txfm && is_inter) { rd = java.lang.StringIndexOutOfBoundsException: Range [27, 12) out of bounds for l |