/*
* Copyright (c) 2016, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <assert.h>
#include <smmintrin.h>
/* SSE4.1 */
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
#include "av1/common/av1_inv_txfm1d_cfg.h"
#include "av1/common/idct.h"
#include "av1/common/x86/av1_inv_txfm_ssse3.h"
#include "av1/common/x86/av1_txfm_sse2.h"
#include "av1/common/x86/av1_txfm_sse4.h"
#include "av1/common/x86/highbd_txfm_utility_sse4.h"
static inline __m128i highbd_clamp_epi16(__m128i u,
int bd) {
const __m128i zero = _mm_setzero_si128();
const __m128i one = _mm_set1_epi16(1);
const __m128i max = _mm_sub_epi16(_mm_slli_epi16(one, bd), one);
__m128i clamped, mask;
mask = _mm_cmpgt_epi16(u, max);
clamped = _mm_andnot_si128(mask, u);
mask = _mm_and_si128(mask, max);
clamped = _mm_or_si128(mask, clamped);
mask = _mm_cmpgt_epi16(clamped, zero);
clamped = _mm_and_si128(clamped, mask);
return clamped;
}
static inline void round_shift_4x4(__m128i *in,
int shift) {
if (shift != 0) {
__m128i rnding = _mm_set1_epi32(1 << (shift - 1));
in[0] = _mm_add_epi32(in[0], rnding);
in[1] = _mm_add_epi32(in[1], rnding);
in[2] = _mm_add_epi32(in[2], rnding);
in[3] = _mm_add_epi32(in[3], rnding);
in[0] = _mm_srai_epi32(in[0], shift);
in[1] = _mm_srai_epi32(in[1], shift);
in[2] = _mm_srai_epi32(in[2], shift);
in[3] = _mm_srai_epi32(in[3], shift);
}
}
static void round_shift_8x8(__m128i *in,
int shift) {
round_shift_4x4(&in[0], shift);
round_shift_4x4(&in[4], shift);
round_shift_4x4(&in[8], shift);
round_shift_4x4(&in[12], shift);
}
static void highbd_clamp_epi32_sse4_1(__m128i *in, __m128i *out,
const __m128i *clamp_lo,
const __m128i *clamp_hi,
int size) {
__m128i a0, a1;
for (
int i = 0; i < size; i += 4) {
a0 = _mm_max_epi32(in[i], *clamp_lo);
out[i] = _mm_min_epi32(a0, *clamp_hi);
a1 = _mm_max_epi32(in[i + 1], *clamp_lo);
out[i + 1] = _mm_min_epi32(a1, *clamp_hi);
a0 = _mm_max_epi32(in[i + 2], *clamp_lo);
out[i + 2] = _mm_min_epi32(a0, *clamp_hi);
a1 = _mm_max_epi32(in[i + 3], *clamp_lo);
out[i + 3] = _mm_min_epi32(a1, *clamp_hi);
}
}
static inline __m128i highbd_get_recon_8x8_sse4_1(
const __m128i pred,
__m128i res0, __m128i res1,
const int bd) {
__m128i x0 = _mm_cvtepi16_epi32(pred);
__m128i x1 = _mm_cvtepi16_epi32(_mm_srli_si128(pred, 8));
__m128i min_clip_val = _mm_setzero_si128();
__m128i max_clip_val = _mm_set1_epi32((1 << bd) - 1);
x0 = _mm_add_epi32(res0, x0);
x1 = _mm_add_epi32(res1, x1);
x0 = _mm_max_epi32(x0, min_clip_val);
x0 = _mm_min_epi32(x0, max_clip_val);
x1 = _mm_max_epi32(x1, min_clip_val);
x1 = _mm_min_epi32(x1, max_clip_val);
x0 = _mm_packus_epi32(x0, x1);
return x0;
}
static inline __m128i highbd_get_recon_4xn_sse4_1(
const __m128i pred,
__m128i res0,
const int bd) {
__m128i x0 = _mm_cvtepi16_epi32(pred);
x0 = _mm_add_epi32(res0, x0);
x0 = _mm_packus_epi32(x0, x0);
x0 = highbd_clamp_epi16(x0, bd);
return x0;
}
static inline void highbd_write_buffer_4xn_sse4_1(__m128i *in, uint16_t *output,
int stride,
int flipud,
int height,
const int bd) {
int j = flipud ? (height - 1) : 0;
const int step = flipud ? -1 : 1;
for (
int i = 0; i < height; ++i, j += step) {
__m128i v = _mm_loadl_epi64((__m128i
const *)(output + i * stride));
__m128i u = highbd_get_recon_4xn_sse4_1(v, in[j], bd);
_mm_storel_epi64((__m128i *)(output + i * stride), u);
}
}
static inline void highbd_write_buffer_8xn_sse4_1(__m128i *in, uint16_t *output,
int stride,
int flipud,
int height,
const int bd) {
int j = flipud ? (height - 1) : 0;
const int step = flipud ? -1 : 1;
for (
int i = 0; i < height; ++i, j += step) {
__m128i v = _mm_loadu_si128((__m128i
const *)(output + i * stride));
__m128i u = highbd_get_recon_8x8_sse4_1(v, in[j], in[j + height], bd);
_mm_storeu_si128((__m128i *)(output + i * stride), u);
}
}
static inline void load_buffer_32bit_input(
const int32_t *in,
int stride,
__m128i *out,
int out_size) {
for (
int i = 0; i < out_size; ++i) {
out[i] = _mm_loadu_si128((
const __m128i *)(in + i * stride));
}
}
static inline void load_buffer_4x4(
const int32_t *coeff, __m128i *in) {
in[0] = _mm_load_si128((
const __m128i *)(coeff + 0));
in[1] = _mm_load_si128((
const __m128i *)(coeff + 4));
in[2] = _mm_load_si128((
const __m128i *)(coeff + 8));
in[3] = _mm_load_si128((
const __m128i *)(coeff + 12));
}
void av1_highbd_iwht4x4_16_add_sse4_1(
const tran_low_t *input, uint8_t *dest8,
int stride,
int bd) {
/* 4-point reversible, orthonormal inverse Walsh-Hadamard in 3.5 adds,
0.5 shifts per pixel. */
__m128i op[4];
uint16_t *dest = CONVERT_TO_SHORTPTR(dest8);
load_buffer_4x4(input, op);
// Shift before-hand.
op[0] = _mm_srai_epi32(op[0], UNIT_QUANT_SHIFT);
op[1] = _mm_srai_epi32(op[1], UNIT_QUANT_SHIFT);
op[2] = _mm_srai_epi32(op[2], UNIT_QUANT_SHIFT);
op[3] = _mm_srai_epi32(op[3], UNIT_QUANT_SHIFT);
for (
int i = 0; i < 2; ++i) {
__m128i a1 = op[0];
__m128i c1 = op[1];
__m128i d1 = op[2];
__m128i b1 = op[3];
a1 = _mm_add_epi32(a1, c1);
// a1 += c1
d1 = _mm_sub_epi32(d1, b1);
// d1 -= b1
__m128i e1 = _mm_sub_epi32(a1, d1);
// e1 = (a1 - d1) >> 1
e1 = _mm_srai_epi32(e1, 1);
b1 = _mm_sub_epi32(e1, b1);
// b1 = e1 - b1
c1 = _mm_sub_epi32(e1, c1);
// c1 = e1 - c1
a1 = _mm_sub_epi32(a1, b1);
// a1 -= b1
d1 = _mm_add_epi32(d1, c1);
// d1 += c1
op[0] = a1;
op[1] = b1;
op[2] = c1;
op[3] = d1;
if (i == 0) {
transpose_32bit_4x4(op, op);
}
}
// Convert to int16_t. The C code checks that we are in range.
op[0] = _mm_packs_epi32(op[0], op[1]);
op[1] = _mm_packs_epi32(op[2], op[3]);
// Load uint16_t.
__m128i dst[2];
__m128i tmp[4];
tmp[0] = _mm_loadl_epi64((
const __m128i *)(dest + 0 * stride));
tmp[1] = _mm_loadl_epi64((
const __m128i *)(dest + 1 * stride));
dst[0] = _mm_unpacklo_epi64(tmp[0], tmp[1]);
tmp[2] = _mm_loadl_epi64((
const __m128i *)(dest + 2 * stride));
tmp[3] = _mm_loadl_epi64((
const __m128i *)(dest + 3 * stride));
dst[1] = _mm_unpacklo_epi64(tmp[2], tmp[3]);
// Add to the previous results.
dst[0] = _mm_add_epi16(dst[0], op[0]);
dst[1] = _mm_add_epi16(dst[1], op[1]);
// Clamp.
dst[0] = highbd_clamp_epi16(dst[0], bd);
dst[1] = highbd_clamp_epi16(dst[1], bd);
// Store.
_mm_storel_epi64((__m128i *)(dest + 0 * stride), dst[0]);
dst[0] = _mm_srli_si128(dst[0], 8);
_mm_storel_epi64((__m128i *)(dest + 1 * stride), dst[0]);
_mm_storel_epi64((__m128i *)(dest + 2 * stride), dst[1]);
dst[1] = _mm_srli_si128(dst[1], 8);
_mm_storel_epi64((__m128i *)(dest + 3 * stride), dst[1]);
}
static void addsub_sse4_1(
const __m128i in0,
const __m128i in1, __m128i *out0,
__m128i *out1,
const __m128i *clamp_lo,
const __m128i *clamp_hi) {
__m128i a0 = _mm_add_epi32(in0, in1);
__m128i a1 = _mm_sub_epi32(in0, in1);
a0 = _mm_max_epi32(a0, *clamp_lo);
a0 = _mm_min_epi32(a0, *clamp_hi);
a1 = _mm_max_epi32(a1, *clamp_lo);
a1 = _mm_min_epi32(a1, *clamp_hi);
*out0 = a0;
*out1 = a1;
}
static void shift_and_clamp_sse4_1(__m128i *in0, __m128i *in1,
const __m128i *clamp_lo,
const __m128i *clamp_hi,
int shift) {
__m128i offset = _mm_set1_epi32((1 << shift) >> 1);
__m128i in0_w_offset = _mm_add_epi32(*in0, offset);
__m128i in1_w_offset = _mm_add_epi32(*in1, offset);
in0_w_offset = _mm_sra_epi32(in0_w_offset, _mm_cvtsi32_si128(shift));
in1_w_offset = _mm_sra_epi32(in1_w_offset, _mm_cvtsi32_si128(shift));
in0_w_offset = _mm_max_epi32(in0_w_offset, *clamp_lo);
in0_w_offset = _mm_min_epi32(in0_w_offset, *clamp_hi);
in1_w_offset = _mm_max_epi32(in1_w_offset, *clamp_lo);
in1_w_offset = _mm_min_epi32(in1_w_offset, *clamp_hi);
*in0 = in0_w_offset;
*in1 = in1_w_offset;
}
static inline void idct32_stage4_sse4_1(
__m128i *bf1,
const __m128i *cospim8,
const __m128i *cospi56,
const __m128i *cospi8,
const __m128i *cospim56,
const __m128i *cospim40,
const __m128i *cospi24,
const __m128i *cospi40,
const __m128i *cospim24,
const __m128i *rounding,
int bit) {
__m128i temp1, temp2;
temp1 = half_btf_sse4_1(cospim8, &bf1[17], cospi56, &bf1[30], rounding, bit);
bf1[30] = half_btf_sse4_1(cospi56, &bf1[17], cospi8, &bf1[30], rounding, bit);
bf1[17] = temp1;
temp2 = half_btf_sse4_1(cospim56, &bf1[18], cospim8, &bf1[29], rounding, bit);
bf1[29] =
half_btf_sse4_1(cospim8, &bf1[18], cospi56, &bf1[29], rounding, bit);
bf1[18] = temp2;
temp1 = half_btf_sse4_1(cospim40, &bf1[21], cospi24, &bf1[26], rounding, bit);
bf1[26] =
half_btf_sse4_1(cospi24, &bf1[21], cospi40, &bf1[26], rounding, bit);
bf1[21] = temp1;
temp2 =
half_btf_sse4_1(cospim24, &bf1[22], cospim40, &bf1[25], rounding, bit);
bf1[25] =
half_btf_sse4_1(cospim40, &bf1[22], cospi24, &bf1[25], rounding, bit);
bf1[22] = temp2;
}
static inline void idct32_stage5_sse4_1(
__m128i *bf1,
const __m128i *cospim16,
const __m128i *cospi48,
const __m128i *cospi16,
const __m128i *cospim48,
const __m128i *clamp_lo,
const __m128i *clamp_hi,
const __m128i *rounding,
int bit) {
__m128i temp1, temp2;
temp1 = half_btf_sse4_1(cospim16, &bf1[9], cospi48, &bf1[14], rounding, bit);
bf1[14] = half_btf_sse4_1(cospi48, &bf1[9], cospi16, &bf1[14], rounding, bit);
bf1[9] = temp1;
temp2 =
half_btf_sse4_1(cospim48, &bf1[10], cospim16, &bf1[13], rounding, bit);
bf1[13] =
half_btf_sse4_1(cospim16, &bf1[10], cospi48, &bf1[13], rounding, bit);
bf1[10] = temp2;
addsub_sse4_1(bf1[16], bf1[19], bf1 + 16, bf1 + 19, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[17], bf1[18], bf1 + 17, bf1 + 18, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[23], bf1[20], bf1 + 23, bf1 + 20, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[22], bf1[21], bf1 + 22, bf1 + 21, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[24], bf1[27], bf1 + 24, bf1 + 27, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[25], bf1[26], bf1 + 25, bf1 + 26, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[31], bf1[28], bf1 + 31, bf1 + 28, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[30], bf1[29], bf1 + 30, bf1 + 29, clamp_lo, clamp_hi);
}
static inline void idct32_stage6_sse4_1(
__m128i *bf1,
const __m128i *cospim32,
const __m128i *cospi32,
const __m128i *cospim16,
const __m128i *cospi48,
const __m128i *cospi16,
const __m128i *cospim48,
const __m128i *clamp_lo,
const __m128i *clamp_hi,
const __m128i *rounding,
int bit) {
__m128i temp1, temp2;
temp1 = half_btf_sse4_1(cospim32, &bf1[5], cospi32, &bf1[6], rounding, bit);
bf1[6] = half_btf_sse4_1(cospi32, &bf1[5], cospi32, &bf1[6], rounding, bit);
bf1[5] = temp1;
addsub_sse4_1(bf1[8], bf1[11], bf1 + 8, bf1 + 11, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[9], bf1[10], bf1 + 9, bf1 + 10, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[15], bf1[12], bf1 + 15, bf1 + 12, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[14], bf1[13], bf1 + 14, bf1 + 13, clamp_lo, clamp_hi);
temp1 = half_btf_sse4_1(cospim16, &bf1[18], cospi48, &bf1[29], rounding, bit);
bf1[29] =
half_btf_sse4_1(cospi48, &bf1[18], cospi16, &bf1[29], rounding, bit);
bf1[18] = temp1;
temp2 = half_btf_sse4_1(cospim16, &bf1[19], cospi48, &bf1[28], rounding, bit);
bf1[28] =
half_btf_sse4_1(cospi48, &bf1[19], cospi16, &bf1[28], rounding, bit);
bf1[19] = temp2;
temp1 =
half_btf_sse4_1(cospim48, &bf1[20], cospim16, &bf1[27], rounding, bit);
bf1[27] =
half_btf_sse4_1(cospim16, &bf1[20], cospi48, &bf1[27], rounding, bit);
bf1[20] = temp1;
temp2 =
half_btf_sse4_1(cospim48, &bf1[21], cospim16, &bf1[26], rounding, bit);
bf1[26] =
half_btf_sse4_1(cospim16, &bf1[21], cospi48, &bf1[26], rounding, bit);
bf1[21] = temp2;
}
static inline void idct32_stage7_sse4_1(__m128i *bf1,
const __m128i *cospim32,
const __m128i *cospi32,
const __m128i *clamp_lo,
const __m128i *clamp_hi,
const __m128i *rounding,
int bit) {
__m128i temp1, temp2;
addsub_sse4_1(bf1[0], bf1[7], bf1 + 0, bf1 + 7, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[1], bf1[6], bf1 + 1, bf1 + 6, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[2], bf1[5], bf1 + 2, bf1 + 5, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[3], bf1[4], bf1 + 3, bf1 + 4, clamp_lo, clamp_hi);
temp1 = half_btf_sse4_1(cospim32, &bf1[10], cospi32, &bf1[13], rounding, bit);
bf1[13] =
half_btf_sse4_1(cospi32, &bf1[10], cospi32, &bf1[13], rounding, bit);
bf1[10] = temp1;
temp2 = half_btf_sse4_1(cospim32, &bf1[11], cospi32, &bf1[12], rounding, bit);
bf1[12] =
half_btf_sse4_1(cospi32, &bf1[11], cospi32, &bf1[12], rounding, bit);
bf1[11] = temp2;
addsub_sse4_1(bf1[16], bf1[23], bf1 + 16, bf1 + 23, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[17], bf1[22], bf1 + 17, bf1 + 22, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[18], bf1[21], bf1 + 18, bf1 + 21, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[19], bf1[20], bf1 + 19, bf1 + 20, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[31], bf1[24], bf1 + 31, bf1 + 24, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[30], bf1[25], bf1 + 30, bf1 + 25, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[29], bf1[26], bf1 + 29, bf1 + 26, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[28], bf1[27], bf1 + 28, bf1 + 27, clamp_lo, clamp_hi);
}
static inline void idct32_stage8_sse4_1(__m128i *bf1,
const __m128i *cospim32,
const __m128i *cospi32,
const __m128i *clamp_lo,
const __m128i *clamp_hi,
const __m128i *rounding,
int bit) {
__m128i temp1, temp2;
addsub_sse4_1(bf1[0], bf1[15], bf1 + 0, bf1 + 15, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[1], bf1[14], bf1 + 1, bf1 + 14, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[2], bf1[13], bf1 + 2, bf1 + 13, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[3], bf1[12], bf1 + 3, bf1 + 12, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[4], bf1[11], bf1 + 4, bf1 + 11, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[5], bf1[10], bf1 + 5, bf1 + 10, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[6], bf1[9], bf1 + 6, bf1 + 9, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[7], bf1[8], bf1 + 7, bf1 + 8, clamp_lo, clamp_hi);
temp1 = half_btf_sse4_1(cospim32, &bf1[20], cospi32, &bf1[27], rounding, bit);
bf1[27] =
half_btf_sse4_1(cospi32, &bf1[20], cospi32, &bf1[27], rounding, bit);
bf1[20] = temp1;
temp2 = half_btf_sse4_1(cospim32, &bf1[21], cospi32, &bf1[26], rounding, bit);
bf1[26] =
half_btf_sse4_1(cospi32, &bf1[21], cospi32, &bf1[26], rounding, bit);
bf1[21] = temp2;
temp1 = half_btf_sse4_1(cospim32, &bf1[22], cospi32, &bf1[25], rounding, bit);
bf1[25] =
half_btf_sse4_1(cospi32, &bf1[22], cospi32, &bf1[25], rounding, bit);
bf1[22] = temp1;
temp2 = half_btf_sse4_1(cospim32, &bf1[23], cospi32, &bf1[24], rounding, bit);
bf1[24] =
half_btf_sse4_1(cospi32, &bf1[23], cospi32, &bf1[24], rounding, bit);
bf1[23] = temp2;
}
static inline void idct32_stage9_sse4_1(__m128i *bf1, __m128i *out,
const int do_cols,
const int bd,
const int out_shift,
const __m128i *clamp_lo,
const __m128i *clamp_hi) {
addsub_sse4_1(bf1[0], bf1[31], out + 0, out + 31, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[1], bf1[30], out + 1, out + 30, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[2], bf1[29], out + 2, out + 29, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[3], bf1[28], out + 3, out + 28, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[4], bf1[27], out + 4, out + 27, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[5], bf1[26], out + 5, out + 26, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[6], bf1[25], out + 6, out + 25, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[7], bf1[24], out + 7, out + 24, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[8], bf1[23], out + 8, out + 23, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[9], bf1[22], out + 9, out + 22, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[10], bf1[21], out + 10, out + 21, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[11], bf1[20], out + 11, out + 20, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[12], bf1[19], out + 12, out + 19, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[13], bf1[18], out + 13, out + 18, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[14], bf1[17], out + 14, out + 17, clamp_lo, clamp_hi);
addsub_sse4_1(bf1[15], bf1[16], out + 15, out + 16, clamp_lo, clamp_hi);
if (!do_cols) {
const int log_range_out = AOMMAX(16, bd + 6);
const __m128i clamp_lo_out = _mm_set1_epi32(-(1 << (log_range_out - 1)));
const __m128i clamp_hi_out = _mm_set1_epi32((1 << (log_range_out - 1)) - 1);
for (
int i = 0; i < 32; i += 8) {
round_shift_4x4(out + i, out_shift);
round_shift_4x4(out + i + 4, out_shift);
}
highbd_clamp_epi32_sse4_1(out, out, &clamp_lo_out, &clamp_hi_out, 32);
}
}
static void neg_shift_sse4_1(
const __m128i in0,
const __m128i in1,
__m128i *out0, __m128i *out1,
const __m128i *clamp_lo,
const __m128i *clamp_hi,
int shift) {
__m128i offset = _mm_set1_epi32((1 << shift) >> 1);
__m128i a0 = _mm_add_epi32(offset, in0);
__m128i a1 = _mm_sub_epi32(offset, in1);
a0 = _mm_sra_epi32(a0, _mm_cvtsi32_si128(shift));
a1 = _mm_sra_epi32(a1, _mm_cvtsi32_si128(shift));
a0 = _mm_max_epi32(a0, *clamp_lo);
a0 = _mm_min_epi32(a0, *clamp_hi);
a1 = _mm_max_epi32(a1, *clamp_lo);
a1 = _mm_min_epi32(a1, *clamp_hi);
*out0 = a0;
*out1 = a1;
}
static void idct4x4_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
const int32_t *cospi = cospi_arr(bit);
const __m128i cospi32 = _mm_set1_epi32(cospi[32]);
const __m128i cospi48 = _mm_set1_epi32(cospi[48]);
const __m128i cospi16 = _mm_set1_epi32(cospi[16]);
const __m128i cospim16 = _mm_set1_epi32(-cospi[16]);
const __m128i rnding = _mm_set1_epi32(1 << (bit - 1));
int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8));
__m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
__m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
__m128i u0, u1, u2, u3;
__m128i v0, v1, v2, v3, x, y;
// Stage 0
// Stage 1
// Stage 2
u0 = in[0];
u1 = in[1];
u2 = in[2];
u3 = in[3];
x = _mm_mullo_epi32(u0, cospi32);
y = _mm_mullo_epi32(u2, cospi32);
v0 = _mm_add_epi32(x, y);
v0 = _mm_add_epi32(v0, rnding);
v0 = _mm_srai_epi32(v0, bit);
v1 = _mm_sub_epi32(x, y);
v1 = _mm_add_epi32(v1, rnding);
v1 = _mm_srai_epi32(v1, bit);
x = _mm_mullo_epi32(u1, cospi48);
y = _mm_mullo_epi32(u3, cospim16);
v2 = _mm_add_epi32(x, y);
v2 = _mm_add_epi32(v2, rnding);
v2 = _mm_srai_epi32(v2, bit);
x = _mm_mullo_epi32(u1, cospi16);
y = _mm_mullo_epi32(u3, cospi48);
v3 = _mm_add_epi32(x, y);
v3 = _mm_add_epi32(v3, rnding);
v3 = _mm_srai_epi32(v3, bit);
// Stage 3
addsub_sse4_1(v0, v3, out + 0, out + 3, &clamp_lo, &clamp_hi);
addsub_sse4_1(v1, v2, out + 1, out + 2, &clamp_lo, &clamp_hi);
if (!do_cols) {
log_range = AOMMAX(16, bd + 6);
clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
shift_and_clamp_sse4_1(out + 0, out + 3, &clamp_lo, &clamp_hi, out_shift);
shift_and_clamp_sse4_1(out + 1, out + 2, &clamp_lo, &clamp_hi, out_shift);
}
}
static void iadst4x4_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
const int32_t *sinpi = sinpi_arr(bit);
const __m128i zero = _mm_setzero_si128();
__m128i rnding = _mm_set1_epi32(1 << (bit + 4 - 1));
rnding = _mm_unpacklo_epi32(rnding, zero);
const __m128i mul = _mm_set1_epi32(1 << 4);
const __m128i sinpi1 = _mm_set1_epi32((
int)sinpi[1]);
const __m128i sinpi2 = _mm_set1_epi32((
int)sinpi[2]);
const __m128i sinpi3 = _mm_set1_epi32((
int)sinpi[3]);
const __m128i sinpi4 = _mm_set1_epi32((
int)sinpi[4]);
__m128i t;
__m128i s0, s1, s2, s3, s4, s5, s6, s7;
__m128i x0, x1, x2, x3;
__m128i u0, u1, u2, u3;
__m128i u0_low, u1_low, u2_low, u3_low;
__m128i u0_high, u1_high, u2_high, u3_high;
x0 = in[0];
x1 = in[1];
x2 = in[2];
x3 = in[3];
s0 = _mm_mullo_epi32(x0, sinpi1);
s1 = _mm_mullo_epi32(x0, sinpi2);
s2 = _mm_mullo_epi32(x1, sinpi3);
s3 = _mm_mullo_epi32(x2, sinpi4);
s4 = _mm_mullo_epi32(x2, sinpi1);
s5 = _mm_mullo_epi32(x3, sinpi2);
s6 = _mm_mullo_epi32(x3, sinpi4);
t = _mm_sub_epi32(x0, x2);
s7 = _mm_add_epi32(t, x3);
t = _mm_add_epi32(s0, s3);
s0 = _mm_add_epi32(t, s5);
t = _mm_sub_epi32(s1, s4);
s1 = _mm_sub_epi32(t, s6);
s3 = s2;
s2 = _mm_mullo_epi32(s7, sinpi3);
u0 = _mm_add_epi32(s0, s3);
u1 = _mm_add_epi32(s1, s3);
u2 = s2;
t = _mm_add_epi32(s0, s1);
u3 = _mm_sub_epi32(t, s3);
// u0
u0_low = _mm_mul_epi32(u0, mul);
u0_low = _mm_add_epi64(u0_low, rnding);
u0 = _mm_srli_si128(u0, 4);
u0_high = _mm_mul_epi32(u0, mul);
u0_high = _mm_add_epi64(u0_high, rnding);
u0_low = _mm_srli_si128(u0_low, 2);
u0_high = _mm_srli_si128(u0_high, 2);
u0 = _mm_unpacklo_epi32(u0_low, u0_high);
u0_high = _mm_unpackhi_epi32(u0_low, u0_high);
u0 = _mm_unpacklo_epi64(u0, u0_high);
// u1
u1_low = _mm_mul_epi32(u1, mul);
u1_low = _mm_add_epi64(u1_low, rnding);
u1 = _mm_srli_si128(u1, 4);
u1_high = _mm_mul_epi32(u1, mul);
u1_high = _mm_add_epi64(u1_high, rnding);
u1_low = _mm_srli_si128(u1_low, 2);
u1_high = _mm_srli_si128(u1_high, 2);
u1 = _mm_unpacklo_epi32(u1_low, u1_high);
u1_high = _mm_unpackhi_epi32(u1_low, u1_high);
u1 = _mm_unpacklo_epi64(u1, u1_high);
// u2
u2_low = _mm_mul_epi32(u2, mul);
u2_low = _mm_add_epi64(u2_low, rnding);
u2 = _mm_srli_si128(u2, 4);
u2_high = _mm_mul_epi32(u2, mul);
u2_high = _mm_add_epi64(u2_high, rnding);
u2_low = _mm_srli_si128(u2_low, 2);
u2_high = _mm_srli_si128(u2_high, 2);
u2 = _mm_unpacklo_epi32(u2_low, u2_high);
u2_high = _mm_unpackhi_epi32(u2_low, u2_high);
u2 = _mm_unpacklo_epi64(u2, u2_high);
// u3
u3_low = _mm_mul_epi32(u3, mul);
u3_low = _mm_add_epi64(u3_low, rnding);
u3 = _mm_srli_si128(u3, 4);
u3_high = _mm_mul_epi32(u3, mul);
u3_high = _mm_add_epi64(u3_high, rnding);
u3_low = _mm_srli_si128(u3_low, 2);
u3_high = _mm_srli_si128(u3_high, 2);
u3 = _mm_unpacklo_epi32(u3_low, u3_high);
u3_high = _mm_unpackhi_epi32(u3_low, u3_high);
u3 = _mm_unpacklo_epi64(u3, u3_high);
out[0] = u0;
out[1] = u1;
out[2] = u2;
out[3] = u3;
if (!do_cols) {
const int log_range = AOMMAX(16, bd + 6);
const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
round_shift_4x4(out, out_shift);
highbd_clamp_epi32_sse4_1(out, out, &clamp_lo, &clamp_hi, 4);
}
}
static void write_buffer_4x4(__m128i *in, uint16_t *output,
int stride,
int fliplr,
int flipud,
int shift,
int bd) {
const __m128i zero = _mm_setzero_si128();
__m128i u0, u1, u2, u3;
__m128i v0, v1, v2, v3;
round_shift_4x4(in, shift);
v0 = _mm_loadl_epi64((__m128i
const *)(output + 0 * stride));
v1 = _mm_loadl_epi64((__m128i
const *)(output + 1 * stride));
v2 = _mm_loadl_epi64((__m128i
const *)(output + 2 * stride));
v3 = _mm_loadl_epi64((__m128i
const *)(output + 3 * stride));
v0 = _mm_unpacklo_epi16(v0, zero);
v1 = _mm_unpacklo_epi16(v1, zero);
v2 = _mm_unpacklo_epi16(v2, zero);
v3 = _mm_unpacklo_epi16(v3, zero);
if (fliplr) {
in[0] = _mm_shuffle_epi32(in[0], 0x1B);
in[1] = _mm_shuffle_epi32(in[1], 0x1B);
in[2] = _mm_shuffle_epi32(in[2], 0x1B);
in[3] = _mm_shuffle_epi32(in[3], 0x1B);
}
if (flipud) {
u0 = _mm_add_epi32(in[3], v0);
u1 = _mm_add_epi32(in[2], v1);
u2 = _mm_add_epi32(in[1], v2);
u3 = _mm_add_epi32(in[0], v3);
}
else {
u0 = _mm_add_epi32(in[0], v0);
u1 = _mm_add_epi32(in[1], v1);
u2 = _mm_add_epi32(in[2], v2);
u3 = _mm_add_epi32(in[3], v3);
}
v0 = _mm_packus_epi32(u0, u1);
v2 = _mm_packus_epi32(u2, u3);
u0 = highbd_clamp_epi16(v0, bd);
u2 = highbd_clamp_epi16(v2, bd);
v0 = _mm_unpacklo_epi64(u0, u0);
v1 = _mm_unpackhi_epi64(u0, u0);
v2 = _mm_unpacklo_epi64(u2, u2);
v3 = _mm_unpackhi_epi64(u2, u2);
_mm_storel_epi64((__m128i *)(output + 0 * stride), v0);
_mm_storel_epi64((__m128i *)(output + 1 * stride), v1);
_mm_storel_epi64((__m128i *)(output + 2 * stride), v2);
_mm_storel_epi64((__m128i *)(output + 3 * stride), v3);
}
static void iidentity4_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
(
void)bit;
__m128i zero = _mm_setzero_si128();
__m128i fact = _mm_set1_epi32(NewSqrt2);
__m128i offset = _mm_set1_epi32(1 << (NewSqrt2Bits - 1));
__m128i a0_low, a1_low;
__m128i a0_high, a1_high;
offset = _mm_unpacklo_epi32(offset, zero);
for (
int i = 0; i < 4; i++) {
a0_low = _mm_mul_epi32(in[i], fact);
a0_low = _mm_add_epi32(a0_low, offset);
a0_low = _mm_srli_epi64(a0_low, NewSqrt2Bits);
a0_high = _mm_srli_si128(in[i], 4);
a0_high = _mm_mul_epi32(a0_high, fact);
a0_high = _mm_add_epi32(a0_high, offset);
a0_high = _mm_srli_epi64(a0_high, NewSqrt2Bits);
a1_low = _mm_unpacklo_epi32(a0_low, a0_high);
a1_high = _mm_unpackhi_epi32(a0_low, a0_high);
out[i] = _mm_unpacklo_epi64(a1_low, a1_high);
}
if (!do_cols) {
const int log_range = AOMMAX(16, bd + 6);
const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
round_shift_4x4(out, out_shift);
highbd_clamp_epi32_sse4_1(out, out, &clamp_lo, &clamp_hi, 4);
}
}
void av1_inv_txfm2d_add_4x4_sse4_1(
const int32_t *input, uint16_t *output,
int stride, TX_TYPE tx_type,
int bd) {
__m128i in[4];
const int8_t *shift = av1_inv_txfm_shift_ls[TX_4X4];
switch (tx_type) {
case DCT_DCT:
load_buffer_4x4(input, in);
idct4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
idct4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case ADST_DCT:
load_buffer_4x4(input, in);
idct4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case DCT_ADST:
load_buffer_4x4(input, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
idct4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case ADST_ADST:
load_buffer_4x4(input, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case FLIPADST_DCT:
load_buffer_4x4(input, in);
idct4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 1, -shift[1], bd);
break;
case DCT_FLIPADST:
load_buffer_4x4(input, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
idct4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 1, 0, -shift[1], bd);
break;
case FLIPADST_FLIPADST:
load_buffer_4x4(input, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 1, 1, -shift[1], bd);
break;
case ADST_FLIPADST:
load_buffer_4x4(input, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 1, 0, -shift[1], bd);
break;
case FLIPADST_ADST:
load_buffer_4x4(input, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 1, -shift[1], bd);
break;
case IDTX:
load_buffer_4x4(input, in);
iidentity4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iidentity4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case V_DCT:
load_buffer_4x4(input, in);
iidentity4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
idct4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case H_DCT:
load_buffer_4x4(input, in);
idct4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iidentity4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case V_ADST:
load_buffer_4x4(input, in);
iidentity4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case H_ADST:
load_buffer_4x4(input, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iidentity4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 0, -shift[1], bd);
break;
case V_FLIPADST:
load_buffer_4x4(input, in);
iidentity4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 0, 1, -shift[1], bd);
break;
case H_FLIPADST:
load_buffer_4x4(input, in);
iadst4x4_sse4_1(in, in, INV_COS_BIT, 0, bd, 0);
transpose_32bit_4x4(in, in);
iidentity4_sse4_1(in, in, INV_COS_BIT, 1, bd, 0);
write_buffer_4x4(in, output, stride, 1, 0, -shift[1], bd);
break;
default: assert(0);
}
}
// 8x8
static void load_buffer_8x8(
const int32_t *coeff, __m128i *in) {
in[0] = _mm_load_si128((
const __m128i *)(coeff + 0));
in[1] = _mm_load_si128((
const __m128i *)(coeff + 4));
in[2] = _mm_load_si128((
const __m128i *)(coeff + 8));
in[3] = _mm_load_si128((
const __m128i *)(coeff + 12));
in[4] = _mm_load_si128((
const __m128i *)(coeff + 16));
in[5] = _mm_load_si128((
const __m128i *)(coeff + 20));
in[6] = _mm_load_si128((
const __m128i *)(coeff + 24));
in[7] = _mm_load_si128((
const __m128i *)(coeff + 28));
in[8] = _mm_load_si128((
const __m128i *)(coeff + 32));
in[9] = _mm_load_si128((
const __m128i *)(coeff + 36));
in[10] = _mm_load_si128((
const __m128i *)(coeff + 40));
in[11] = _mm_load_si128((
const __m128i *)(coeff + 44));
in[12] = _mm_load_si128((
const __m128i *)(coeff + 48));
in[13] = _mm_load_si128((
const __m128i *)(coeff + 52));
in[14] = _mm_load_si128((
const __m128i *)(coeff + 56));
in[15] = _mm_load_si128((
const __m128i *)(coeff + 60));
}
static void idct8x8_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
const int32_t *cospi = cospi_arr(bit);
const __m128i cospi56 = _mm_set1_epi32(cospi[56]);
const __m128i cospim8 = _mm_set1_epi32(-cospi[8]);
const __m128i cospi24 = _mm_set1_epi32(cospi[24]);
const __m128i cospim40 = _mm_set1_epi32(-cospi[40]);
const __m128i cospi40 = _mm_set1_epi32(cospi[40]);
const __m128i cospi8 = _mm_set1_epi32(cospi[8]);
const __m128i cospi32 = _mm_set1_epi32(cospi[32]);
const __m128i cospi48 = _mm_set1_epi32(cospi[48]);
const __m128i cospim16 = _mm_set1_epi32(-cospi[16]);
const __m128i cospi16 = _mm_set1_epi32(cospi[16]);
const __m128i rnding = _mm_set1_epi32(1 << (bit - 1));
const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8));
const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
__m128i u0, u1, u2, u3, u4, u5, u6, u7;
__m128i v0, v1, v2, v3, v4, v5, v6, v7;
__m128i x, y;
int col;
// Note:
// Even column: 0, 2, ..., 14
// Odd column: 1, 3, ..., 15
// one even column plus one odd column constructs one row (8 coeffs)
// total we have 8 rows (8x8).
for (col = 0; col < 2; ++col) {
// stage 0
// stage 1
// stage 2
u0 = in[0 * 2 + col];
u1 = in[4 * 2 + col];
u2 = in[2 * 2 + col];
u3 = in[6 * 2 + col];
x = _mm_mullo_epi32(in[1 * 2 + col], cospi56);
y = _mm_mullo_epi32(in[7 * 2 + col], cospim8);
u4 = _mm_add_epi32(x, y);
u4 = _mm_add_epi32(u4, rnding);
u4 = _mm_srai_epi32(u4, bit);
x = _mm_mullo_epi32(in[1 * 2 + col], cospi8);
y = _mm_mullo_epi32(in[7 * 2 + col], cospi56);
u7 = _mm_add_epi32(x, y);
u7 = _mm_add_epi32(u7, rnding);
u7 = _mm_srai_epi32(u7, bit);
x = _mm_mullo_epi32(in[5 * 2 + col], cospi24);
y = _mm_mullo_epi32(in[3 * 2 + col], cospim40);
u5 = _mm_add_epi32(x, y);
u5 = _mm_add_epi32(u5, rnding);
u5 = _mm_srai_epi32(u5, bit);
x = _mm_mullo_epi32(in[5 * 2 + col], cospi40);
y = _mm_mullo_epi32(in[3 * 2 + col], cospi24);
u6 = _mm_add_epi32(x, y);
u6 = _mm_add_epi32(u6, rnding);
u6 = _mm_srai_epi32(u6, bit);
// stage 3
x = _mm_mullo_epi32(u0, cospi32);
y = _mm_mullo_epi32(u1, cospi32);
v0 = _mm_add_epi32(x, y);
v0 = _mm_add_epi32(v0, rnding);
v0 = _mm_srai_epi32(v0, bit);
v1 = _mm_sub_epi32(x, y);
v1 = _mm_add_epi32(v1, rnding);
v1 = _mm_srai_epi32(v1, bit);
x = _mm_mullo_epi32(u2, cospi48);
y = _mm_mullo_epi32(u3, cospim16);
v2 = _mm_add_epi32(x, y);
v2 = _mm_add_epi32(v2, rnding);
v2 = _mm_srai_epi32(v2, bit);
x = _mm_mullo_epi32(u2, cospi16);
y = _mm_mullo_epi32(u3, cospi48);
v3 = _mm_add_epi32(x, y);
v3 = _mm_add_epi32(v3, rnding);
v3 = _mm_srai_epi32(v3, bit);
addsub_sse4_1(u4, u5, &v4, &v5, &clamp_lo, &clamp_hi);
addsub_sse4_1(u7, u6, &v7, &v6, &clamp_lo, &clamp_hi);
// stage 4
addsub_sse4_1(v0, v3, &u0, &u3, &clamp_lo, &clamp_hi);
addsub_sse4_1(v1, v2, &u1, &u2, &clamp_lo, &clamp_hi);
u4 = v4;
u7 = v7;
x = _mm_mullo_epi32(v5, cospi32);
y = _mm_mullo_epi32(v6, cospi32);
u6 = _mm_add_epi32(y, x);
u6 = _mm_add_epi32(u6, rnding);
u6 = _mm_srai_epi32(u6, bit);
u5 = _mm_sub_epi32(y, x);
u5 = _mm_add_epi32(u5, rnding);
u5 = _mm_srai_epi32(u5, bit);
// stage 5
addsub_sse4_1(u0, u7, out + 0 * 2 + col, out + 7 * 2 + col, &clamp_lo,
&clamp_hi);
addsub_sse4_1(u1, u6, out + 1 * 2 + col, out + 6 * 2 + col, &clamp_lo,
&clamp_hi);
addsub_sse4_1(u2, u5, out + 2 * 2 + col, out + 5 * 2 + col, &clamp_lo,
&clamp_hi);
addsub_sse4_1(u3, u4, out + 3 * 2 + col, out + 4 * 2 + col, &clamp_lo,
&clamp_hi);
}
if (!do_cols) {
const int log_range_out = AOMMAX(16, bd + 6);
const __m128i clamp_lo_out = _mm_set1_epi32(-(1 << (log_range_out - 1)));
const __m128i clamp_hi_out = _mm_set1_epi32((1 << (log_range_out - 1)) - 1);
round_shift_8x8(out, out_shift);
highbd_clamp_epi32_sse4_1(out, out, &clamp_lo_out, &clamp_hi_out, 16);
}
}
static void iadst8x8_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
const int32_t *cospi = cospi_arr(bit);
const __m128i cospi4 = _mm_set1_epi32(cospi[4]);
const __m128i cospi60 = _mm_set1_epi32(cospi[60]);
const __m128i cospi20 = _mm_set1_epi32(cospi[20]);
const __m128i cospi44 = _mm_set1_epi32(cospi[44]);
const __m128i cospi36 = _mm_set1_epi32(cospi[36]);
const __m128i cospi28 = _mm_set1_epi32(cospi[28]);
const __m128i cospi52 = _mm_set1_epi32(cospi[52]);
const __m128i cospi12 = _mm_set1_epi32(cospi[12]);
const __m128i cospi16 = _mm_set1_epi32(cospi[16]);
const __m128i cospi48 = _mm_set1_epi32(cospi[48]);
const __m128i cospim48 = _mm_set1_epi32(-cospi[48]);
const __m128i cospi32 = _mm_set1_epi32(cospi[32]);
const __m128i rnding = _mm_set1_epi32(1 << (bit - 1));
const __m128i kZero = _mm_setzero_si128();
const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8));
const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
__m128i u[8], v[8], x;
// Even 8 points: 0, 2, ..., 14
// stage 0
// stage 1
// stage 2
// (1)
u[0] = _mm_mullo_epi32(in[14], cospi4);
x = _mm_mullo_epi32(in[0], cospi60);
u[0] = _mm_add_epi32(u[0], x);
u[0] = _mm_add_epi32(u[0], rnding);
u[0] = _mm_srai_epi32(u[0], bit);
u[1] = _mm_mullo_epi32(in[14], cospi60);
x = _mm_mullo_epi32(in[0], cospi4);
u[1] = _mm_sub_epi32(u[1], x);
u[1] = _mm_add_epi32(u[1], rnding);
u[1] = _mm_srai_epi32(u[1], bit);
// (2)
u[2] = _mm_mullo_epi32(in[10], cospi20);
x = _mm_mullo_epi32(in[4], cospi44);
u[2] = _mm_add_epi32(u[2], x);
u[2] = _mm_add_epi32(u[2], rnding);
u[2] = _mm_srai_epi32(u[2], bit);
u[3] = _mm_mullo_epi32(in[10], cospi44);
x = _mm_mullo_epi32(in[4], cospi20);
u[3] = _mm_sub_epi32(u[3], x);
u[3] = _mm_add_epi32(u[3], rnding);
u[3] = _mm_srai_epi32(u[3], bit);
// (3)
u[4] = _mm_mullo_epi32(in[6], cospi36);
x = _mm_mullo_epi32(in[8], cospi28);
u[4] = _mm_add_epi32(u[4], x);
u[4] = _mm_add_epi32(u[4], rnding);
u[4] = _mm_srai_epi32(u[4], bit);
u[5] = _mm_mullo_epi32(in[6], cospi28);
x = _mm_mullo_epi32(in[8], cospi36);
u[5] = _mm_sub_epi32(u[5], x);
u[5] = _mm_add_epi32(u[5], rnding);
u[5] = _mm_srai_epi32(u[5], bit);
// (4)
u[6] = _mm_mullo_epi32(in[2], cospi52);
x = _mm_mullo_epi32(in[12], cospi12);
u[6] = _mm_add_epi32(u[6], x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_mullo_epi32(in[2], cospi12);
x = _mm_mullo_epi32(in[12], cospi52);
u[7] = _mm_sub_epi32(u[7], x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 3
addsub_sse4_1(u[0], u[4], &v[0], &v[4], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[1], u[5], &v[1], &v[5], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[2], u[6], &v[2], &v[6], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[3], u[7], &v[3], &v[7], &clamp_lo, &clamp_hi);
// stage 4
u[0] = v[0];
u[1] = v[1];
u[2] = v[2];
u[3] = v[3];
u[4] = _mm_mullo_epi32(v[4], cospi16);
x = _mm_mullo_epi32(v[5], cospi48);
u[4] = _mm_add_epi32(u[4], x);
u[4] = _mm_add_epi32(u[4], rnding);
u[4] = _mm_srai_epi32(u[4], bit);
u[5] = _mm_mullo_epi32(v[4], cospi48);
x = _mm_mullo_epi32(v[5], cospi16);
u[5] = _mm_sub_epi32(u[5], x);
u[5] = _mm_add_epi32(u[5], rnding);
u[5] = _mm_srai_epi32(u[5], bit);
u[6] = _mm_mullo_epi32(v[6], cospim48);
x = _mm_mullo_epi32(v[7], cospi16);
u[6] = _mm_add_epi32(u[6], x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_mullo_epi32(v[6], cospi16);
x = _mm_mullo_epi32(v[7], cospim48);
u[7] = _mm_sub_epi32(u[7], x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 5
addsub_sse4_1(u[0], u[2], &v[0], &v[2], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[1], u[3], &v[1], &v[3], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[4], u[6], &v[4], &v[6], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[5], u[7], &v[5], &v[7], &clamp_lo, &clamp_hi);
// stage 6
u[0] = v[0];
u[1] = v[1];
u[4] = v[4];
u[5] = v[5];
v[0] = _mm_mullo_epi32(v[2], cospi32);
x = _mm_mullo_epi32(v[3], cospi32);
u[2] = _mm_add_epi32(v[0], x);
u[2] = _mm_add_epi32(u[2], rnding);
u[2] = _mm_srai_epi32(u[2], bit);
u[3] = _mm_sub_epi32(v[0], x);
u[3] = _mm_add_epi32(u[3], rnding);
u[3] = _mm_srai_epi32(u[3], bit);
v[0] = _mm_mullo_epi32(v[6], cospi32);
x = _mm_mullo_epi32(v[7], cospi32);
u[6] = _mm_add_epi32(v[0], x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_sub_epi32(v[0], x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 7
if (do_cols) {
out[0] = u[0];
out[2] = _mm_sub_epi32(kZero, u[4]);
out[4] = u[6];
out[6] = _mm_sub_epi32(kZero, u[2]);
out[8] = u[3];
out[10] = _mm_sub_epi32(kZero, u[7]);
out[12] = u[5];
out[14] = _mm_sub_epi32(kZero, u[1]);
}
else {
const int log_range_out = AOMMAX(16, bd + 6);
const __m128i clamp_lo_out = _mm_set1_epi32(-(1 << (log_range_out - 1)));
const __m128i clamp_hi_out = _mm_set1_epi32((1 << (log_range_out - 1)) - 1);
neg_shift_sse4_1(u[0], u[4], out + 0, out + 2, &clamp_lo_out, &clamp_hi_out,
out_shift);
neg_shift_sse4_1(u[6], u[2], out + 4, out + 6, &clamp_lo_out, &clamp_hi_out,
out_shift);
neg_shift_sse4_1(u[3], u[7], out + 8, out + 10, &clamp_lo_out,
&clamp_hi_out, out_shift);
neg_shift_sse4_1(u[5], u[1], out + 12, out + 14, &clamp_lo_out,
&clamp_hi_out, out_shift);
}
// Odd 8 points: 1, 3, ..., 15
// stage 0
// stage 1
// stage 2
// (1)
u[0] = _mm_mullo_epi32(in[15], cospi4);
x = _mm_mullo_epi32(in[1], cospi60);
u[0] = _mm_add_epi32(u[0], x);
u[0] = _mm_add_epi32(u[0], rnding);
u[0] = _mm_srai_epi32(u[0], bit);
u[1] = _mm_mullo_epi32(in[15], cospi60);
x = _mm_mullo_epi32(in[1], cospi4);
u[1] = _mm_sub_epi32(u[1], x);
u[1] = _mm_add_epi32(u[1], rnding);
u[1] = _mm_srai_epi32(u[1], bit);
// (2)
u[2] = _mm_mullo_epi32(in[11], cospi20);
x = _mm_mullo_epi32(in[5], cospi44);
u[2] = _mm_add_epi32(u[2], x);
u[2] = _mm_add_epi32(u[2], rnding);
u[2] = _mm_srai_epi32(u[2], bit);
u[3] = _mm_mullo_epi32(in[11], cospi44);
x = _mm_mullo_epi32(in[5], cospi20);
u[3] = _mm_sub_epi32(u[3], x);
u[3] = _mm_add_epi32(u[3], rnding);
u[3] = _mm_srai_epi32(u[3], bit);
// (3)
u[4] = _mm_mullo_epi32(in[7], cospi36);
x = _mm_mullo_epi32(in[9], cospi28);
u[4] = _mm_add_epi32(u[4], x);
u[4] = _mm_add_epi32(u[4], rnding);
u[4] = _mm_srai_epi32(u[4], bit);
u[5] = _mm_mullo_epi32(in[7], cospi28);
x = _mm_mullo_epi32(in[9], cospi36);
u[5] = _mm_sub_epi32(u[5], x);
u[5] = _mm_add_epi32(u[5], rnding);
u[5] = _mm_srai_epi32(u[5], bit);
// (4)
u[6] = _mm_mullo_epi32(in[3], cospi52);
x = _mm_mullo_epi32(in[13], cospi12);
u[6] = _mm_add_epi32(u[6], x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_mullo_epi32(in[3], cospi12);
x = _mm_mullo_epi32(in[13], cospi52);
u[7] = _mm_sub_epi32(u[7], x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 3
addsub_sse4_1(u[0], u[4], &v[0], &v[4], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[1], u[5], &v[1], &v[5], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[2], u[6], &v[2], &v[6], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[3], u[7], &v[3], &v[7], &clamp_lo, &clamp_hi);
// stage 4
u[0] = v[0];
u[1] = v[1];
u[2] = v[2];
u[3] = v[3];
u[4] = _mm_mullo_epi32(v[4], cospi16);
x = _mm_mullo_epi32(v[5], cospi48);
u[4] = _mm_add_epi32(u[4], x);
u[4] = _mm_add_epi32(u[4], rnding);
u[4] = _mm_srai_epi32(u[4], bit);
u[5] = _mm_mullo_epi32(v[4], cospi48);
x = _mm_mullo_epi32(v[5], cospi16);
u[5] = _mm_sub_epi32(u[5], x);
u[5] = _mm_add_epi32(u[5], rnding);
u[5] = _mm_srai_epi32(u[5], bit);
u[6] = _mm_mullo_epi32(v[6], cospim48);
x = _mm_mullo_epi32(v[7], cospi16);
u[6] = _mm_add_epi32(u[6], x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_mullo_epi32(v[6], cospi16);
x = _mm_mullo_epi32(v[7], cospim48);
u[7] = _mm_sub_epi32(u[7], x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 5
addsub_sse4_1(u[0], u[2], &v[0], &v[2], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[1], u[3], &v[1], &v[3], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[4], u[6], &v[4], &v[6], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[5], u[7], &v[5], &v[7], &clamp_lo, &clamp_hi);
// stage 6
u[0] = v[0];
u[1] = v[1];
u[4] = v[4];
u[5] = v[5];
v[0] = _mm_mullo_epi32(v[2], cospi32);
x = _mm_mullo_epi32(v[3], cospi32);
u[2] = _mm_add_epi32(v[0], x);
u[2] = _mm_add_epi32(u[2], rnding);
u[2] = _mm_srai_epi32(u[2], bit);
u[3] = _mm_sub_epi32(v[0], x);
u[3] = _mm_add_epi32(u[3], rnding);
u[3] = _mm_srai_epi32(u[3], bit);
v[0] = _mm_mullo_epi32(v[6], cospi32);
x = _mm_mullo_epi32(v[7], cospi32);
u[6] = _mm_add_epi32(v[0], x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_sub_epi32(v[0], x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 7
if (do_cols) {
out[1] = u[0];
out[3] = _mm_sub_epi32(kZero, u[4]);
out[5] = u[6];
out[7] = _mm_sub_epi32(kZero, u[2]);
out[9] = u[3];
out[11] = _mm_sub_epi32(kZero, u[7]);
out[13] = u[5];
out[15] = _mm_sub_epi32(kZero, u[1]);
}
else {
const int log_range_out = AOMMAX(16, bd + 6);
const __m128i clamp_lo_out = _mm_set1_epi32(-(1 << (log_range_out - 1)));
const __m128i clamp_hi_out = _mm_set1_epi32((1 << (log_range_out - 1)) - 1);
neg_shift_sse4_1(u[0], u[4], out + 1, out + 3, &clamp_lo_out, &clamp_hi_out,
out_shift);
neg_shift_sse4_1(u[6], u[2], out + 5, out + 7, &clamp_lo_out, &clamp_hi_out,
out_shift);
neg_shift_sse4_1(u[3], u[7], out + 9, out + 11, &clamp_lo_out,
&clamp_hi_out, out_shift);
neg_shift_sse4_1(u[5], u[1], out + 13, out + 15, &clamp_lo_out,
&clamp_hi_out, out_shift);
}
}
static void iidentity8_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
(
void)bit;
out[0] = _mm_add_epi32(in[0], in[0]);
out[1] = _mm_add_epi32(in[1], in[1]);
out[2] = _mm_add_epi32(in[2], in[2]);
out[3] = _mm_add_epi32(in[3], in[3]);
out[4] = _mm_add_epi32(in[4], in[4]);
out[5] = _mm_add_epi32(in[5], in[5]);
out[6] = _mm_add_epi32(in[6], in[6]);
out[7] = _mm_add_epi32(in[7], in[7]);
if (!do_cols) {
const int log_range = AOMMAX(16, bd + 6);
const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
round_shift_4x4(out, out_shift);
round_shift_4x4(out + 4, out_shift);
highbd_clamp_epi32_sse4_1(out, out, &clamp_lo, &clamp_hi, 8);
}
}
static __m128i get_recon_8x8(
const __m128i pred, __m128i res_lo, __m128i res_hi,
int fliplr,
int bd) {
__m128i x0, x1;
const __m128i zero = _mm_setzero_si128();
x0 = _mm_unpacklo_epi16(pred, zero);
x1 = _mm_unpackhi_epi16(pred, zero);
if (fliplr) {
res_lo = _mm_shuffle_epi32(res_lo, 0x1B);
res_hi = _mm_shuffle_epi32(res_hi, 0x1B);
x0 = _mm_add_epi32(res_hi, x0);
x1 = _mm_add_epi32(res_lo, x1);
}
else {
x0 = _mm_add_epi32(res_lo, x0);
x1 = _mm_add_epi32(res_hi, x1);
}
x0 = _mm_packus_epi32(x0, x1);
return highbd_clamp_epi16(x0, bd);
}
static void write_buffer_8x8(__m128i *in, uint16_t *output,
int stride,
int fliplr,
int flipud,
int shift,
int bd) {
__m128i u0, u1, u2, u3, u4, u5, u6, u7;
__m128i v0, v1, v2, v3, v4, v5, v6, v7;
round_shift_8x8(in, shift);
v0 = _mm_load_si128((__m128i
const *)(output + 0 * stride));
v1 = _mm_load_si128((__m128i
const *)(output + 1 * stride));
v2 = _mm_load_si128((__m128i
const *)(output + 2 * stride));
v3 = _mm_load_si128((__m128i
const *)(output + 3 * stride));
v4 = _mm_load_si128((__m128i
const *)(output + 4 * stride));
v5 = _mm_load_si128((__m128i
const *)(output + 5 * stride));
v6 = _mm_load_si128((__m128i
const *)(output + 6 * stride));
v7 = _mm_load_si128((__m128i
const *)(output + 7 * stride));
if (flipud) {
u0 = get_recon_8x8(v0, in[14], in[15], fliplr, bd);
u1 = get_recon_8x8(v1, in[12], in[13], fliplr, bd);
u2 = get_recon_8x8(v2, in[10], in[11], fliplr, bd);
u3 = get_recon_8x8(v3, in[8], in[9], fliplr, bd);
u4 = get_recon_8x8(v4, in[6], in[7], fliplr, bd);
u5 = get_recon_8x8(v5, in[4], in[5], fliplr, bd);
u6 = get_recon_8x8(v6, in[2], in[3], fliplr, bd);
u7 = get_recon_8x8(v7, in[0], in[1], fliplr, bd);
}
else {
u0 = get_recon_8x8(v0, in[0], in[1], fliplr, bd);
u1 = get_recon_8x8(v1, in[2], in[3], fliplr, bd);
u2 = get_recon_8x8(v2, in[4], in[5], fliplr, bd);
u3 = get_recon_8x8(v3, in[6], in[7], fliplr, bd);
u4 = get_recon_8x8(v4, in[8], in[9], fliplr, bd);
u5 = get_recon_8x8(v5, in[10], in[11], fliplr, bd);
u6 = get_recon_8x8(v6, in[12], in[13], fliplr, bd);
u7 = get_recon_8x8(v7, in[14], in[15], fliplr, bd);
}
_mm_store_si128((__m128i *)(output + 0 * stride), u0);
_mm_store_si128((__m128i *)(output + 1 * stride), u1);
_mm_store_si128((__m128i *)(output + 2 * stride), u2);
_mm_store_si128((__m128i *)(output + 3 * stride), u3);
_mm_store_si128((__m128i *)(output + 4 * stride), u4);
_mm_store_si128((__m128i *)(output + 5 * stride), u5);
_mm_store_si128((__m128i *)(output + 6 * stride), u6);
_mm_store_si128((__m128i *)(output + 7 * stride), u7);
}
void av1_inv_txfm2d_add_8x8_sse4_1(
const int32_t *input, uint16_t *output,
int stride, TX_TYPE tx_type,
int bd) {
__m128i in[16], out[16];
const int8_t *shift = av1_inv_txfm_shift_ls[TX_8X8];
switch (tx_type) {
case DCT_DCT:
load_buffer_8x8(input, in);
idct8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
idct8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 0, 0, -shift[1], bd);
break;
case DCT_ADST:
load_buffer_8x8(input, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
idct8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 0, 0, -shift[1], bd);
break;
case ADST_DCT:
load_buffer_8x8(input, in);
idct8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 0, 0, -shift[1], bd);
break;
case ADST_ADST:
load_buffer_8x8(input, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 0, 0, -shift[1], bd);
break;
case FLIPADST_DCT:
load_buffer_8x8(input, in);
idct8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 0, 1, -shift[1], bd);
break;
case DCT_FLIPADST:
load_buffer_8x8(input, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
idct8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 1, 0, -shift[1], bd);
break;
case ADST_FLIPADST:
load_buffer_8x8(input, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 1, 0, -shift[1], bd);
break;
case FLIPADST_FLIPADST:
load_buffer_8x8(input, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 1, 1, -shift[1], bd);
break;
case FLIPADST_ADST:
load_buffer_8x8(input, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 0, bd, -shift[0]);
transpose_8x8(out, in);
iadst8x8_sse4_1(in, out, INV_COS_BIT, 1, bd, 0);
write_buffer_8x8(out, output, stride, 0, 1, -shift[1], bd);
break;
default: assert(0);
}
}
static void idct8x8_low1_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
const int32_t *cospi = cospi_arr(bit);
const __m128i cospi32 = _mm_set1_epi32(cospi[32]);
const __m128i rnding = _mm_set1_epi32(1 << (bit - 1));
const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8));
__m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
__m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
__m128i x;
// stage 0
// stage 1
// stage 2
// stage 3
x = _mm_mullo_epi32(in[0], cospi32);
x = _mm_add_epi32(x, rnding);
x = _mm_srai_epi32(x, bit);
// stage 4
// stage 5
if (!do_cols) {
const int log_range_out = AOMMAX(16, bd + 6);
clamp_lo = _mm_set1_epi32(-(1 << (log_range_out - 1)));
clamp_hi = _mm_set1_epi32((1 << (log_range_out - 1)) - 1);
__m128i offset = _mm_set1_epi32((1 << out_shift) >> 1);
x = _mm_add_epi32(x, offset);
x = _mm_sra_epi32(x, _mm_cvtsi32_si128(out_shift));
}
x = _mm_max_epi32(x, clamp_lo);
x = _mm_min_epi32(x, clamp_hi);
out[0] = x;
out[1] = x;
out[2] = x;
out[3] = x;
out[4] = x;
out[5] = x;
out[6] = x;
out[7] = x;
}
static void idct8x8_new_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
const int32_t *cospi = cospi_arr(bit);
const __m128i cospi56 = _mm_set1_epi32(cospi[56]);
const __m128i cospim8 = _mm_set1_epi32(-cospi[8]);
const __m128i cospi24 = _mm_set1_epi32(cospi[24]);
const __m128i cospim40 = _mm_set1_epi32(-cospi[40]);
const __m128i cospi40 = _mm_set1_epi32(cospi[40]);
const __m128i cospi8 = _mm_set1_epi32(cospi[8]);
const __m128i cospi32 = _mm_set1_epi32(cospi[32]);
const __m128i cospi48 = _mm_set1_epi32(cospi[48]);
const __m128i cospim16 = _mm_set1_epi32(-cospi[16]);
const __m128i cospi16 = _mm_set1_epi32(cospi[16]);
const __m128i rnding = _mm_set1_epi32(1 << (bit - 1));
const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8));
const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
__m128i u0, u1, u2, u3, u4, u5, u6, u7;
__m128i v0, v1, v2, v3, v4, v5, v6, v7;
__m128i x, y;
// stage 0
// stage 1
// stage 2
u0 = in[0];
u1 = in[4];
u2 = in[2];
u3 = in[6];
x = _mm_mullo_epi32(in[1], cospi56);
y = _mm_mullo_epi32(in[7], cospim8);
u4 = _mm_add_epi32(x, y);
u4 = _mm_add_epi32(u4, rnding);
u4 = _mm_srai_epi32(u4, bit);
x = _mm_mullo_epi32(in[1], cospi8);
y = _mm_mullo_epi32(in[7], cospi56);
u7 = _mm_add_epi32(x, y);
u7 = _mm_add_epi32(u7, rnding);
u7 = _mm_srai_epi32(u7, bit);
x = _mm_mullo_epi32(in[5], cospi24);
y = _mm_mullo_epi32(in[3], cospim40);
u5 = _mm_add_epi32(x, y);
u5 = _mm_add_epi32(u5, rnding);
u5 = _mm_srai_epi32(u5, bit);
x = _mm_mullo_epi32(in[5], cospi40);
y = _mm_mullo_epi32(in[3], cospi24);
u6 = _mm_add_epi32(x, y);
u6 = _mm_add_epi32(u6, rnding);
u6 = _mm_srai_epi32(u6, bit);
// stage 3
x = _mm_mullo_epi32(u0, cospi32);
y = _mm_mullo_epi32(u1, cospi32);
v0 = _mm_add_epi32(x, y);
v0 = _mm_add_epi32(v0, rnding);
v0 = _mm_srai_epi32(v0, bit);
v1 = _mm_sub_epi32(x, y);
v1 = _mm_add_epi32(v1, rnding);
v1 = _mm_srai_epi32(v1, bit);
x = _mm_mullo_epi32(u2, cospi48);
y = _mm_mullo_epi32(u3, cospim16);
v2 = _mm_add_epi32(x, y);
v2 = _mm_add_epi32(v2, rnding);
v2 = _mm_srai_epi32(v2, bit);
x = _mm_mullo_epi32(u2, cospi16);
y = _mm_mullo_epi32(u3, cospi48);
v3 = _mm_add_epi32(x, y);
v3 = _mm_add_epi32(v3, rnding);
v3 = _mm_srai_epi32(v3, bit);
addsub_sse4_1(u4, u5, &v4, &v5, &clamp_lo, &clamp_hi);
addsub_sse4_1(u7, u6, &v7, &v6, &clamp_lo, &clamp_hi);
// stage 4
addsub_sse4_1(v0, v3, &u0, &u3, &clamp_lo, &clamp_hi);
addsub_sse4_1(v1, v2, &u1, &u2, &clamp_lo, &clamp_hi);
u4 = v4;
u7 = v7;
x = _mm_mullo_epi32(v5, cospi32);
y = _mm_mullo_epi32(v6, cospi32);
u6 = _mm_add_epi32(y, x);
u6 = _mm_add_epi32(u6, rnding);
u6 = _mm_srai_epi32(u6, bit);
u5 = _mm_sub_epi32(y, x);
u5 = _mm_add_epi32(u5, rnding);
u5 = _mm_srai_epi32(u5, bit);
// stage 5
addsub_sse4_1(u0, u7, out + 0, out + 7, &clamp_lo, &clamp_hi);
addsub_sse4_1(u1, u6, out + 1, out + 6, &clamp_lo, &clamp_hi);
addsub_sse4_1(u2, u5, out + 2, out + 5, &clamp_lo, &clamp_hi);
addsub_sse4_1(u3, u4, out + 3, out + 4, &clamp_lo, &clamp_hi);
if (!do_cols) {
const int log_range_out = AOMMAX(16, bd + 6);
const __m128i clamp_lo_out = _mm_set1_epi32(-(1 << (log_range_out - 1)));
const __m128i clamp_hi_out = _mm_set1_epi32((1 << (log_range_out - 1)) - 1);
round_shift_4x4(out, out_shift);
round_shift_4x4(out + 4, out_shift);
highbd_clamp_epi32_sse4_1(out, out, &clamp_lo_out, &clamp_hi_out, 8);
}
}
static void iadst8x8_low1_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
const int32_t *cospi = cospi_arr(bit);
const __m128i cospi4 = _mm_set1_epi32(cospi[4]);
const __m128i cospi60 = _mm_set1_epi32(cospi[60]);
const __m128i cospi16 = _mm_set1_epi32(cospi[16]);
const __m128i cospi48 = _mm_set1_epi32(cospi[48]);
const __m128i cospi32 = _mm_set1_epi32(cospi[32]);
const __m128i rnding = _mm_set1_epi32(1 << (bit - 1));
const __m128i kZero = _mm_setzero_si128();
__m128i u[8], x;
// stage 0
// stage 1
// stage 2
x = _mm_mullo_epi32(in[0], cospi60);
u[0] = _mm_add_epi32(x, rnding);
u[0] = _mm_srai_epi32(u[0], bit);
x = _mm_mullo_epi32(in[0], cospi4);
u[1] = _mm_sub_epi32(kZero, x);
u[1] = _mm_add_epi32(u[1], rnding);
u[1] = _mm_srai_epi32(u[1], bit);
// stage 3
// stage 4
__m128i temp1, temp2;
temp1 = _mm_mullo_epi32(u[0], cospi16);
x = _mm_mullo_epi32(u[1], cospi48);
temp1 = _mm_add_epi32(temp1, x);
temp1 = _mm_add_epi32(temp1, rnding);
temp1 = _mm_srai_epi32(temp1, bit);
u[4] = temp1;
temp2 = _mm_mullo_epi32(u[0], cospi48);
x = _mm_mullo_epi32(u[1], cospi16);
u[5] = _mm_sub_epi32(temp2, x);
u[5] = _mm_add_epi32(u[5], rnding);
u[5] = _mm_srai_epi32(u[5], bit);
// stage 5
// stage 6
temp1 = _mm_mullo_epi32(u[0], cospi32);
x = _mm_mullo_epi32(u[1], cospi32);
u[2] = _mm_add_epi32(temp1, x);
u[2] = _mm_add_epi32(u[2], rnding);
u[2] = _mm_srai_epi32(u[2], bit);
u[3] = _mm_sub_epi32(temp1, x);
u[3] = _mm_add_epi32(u[3], rnding);
u[3] = _mm_srai_epi32(u[3], bit);
temp1 = _mm_mullo_epi32(u[4], cospi32);
x = _mm_mullo_epi32(u[5], cospi32);
u[6] = _mm_add_epi32(temp1, x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_sub_epi32(temp1, x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 7
if (do_cols) {
out[0] = u[0];
out[1] = _mm_sub_epi32(kZero, u[4]);
out[2] = u[6];
out[3] = _mm_sub_epi32(kZero, u[2]);
out[4] = u[3];
out[5] = _mm_sub_epi32(kZero, u[7]);
out[6] = u[5];
out[7] = _mm_sub_epi32(kZero, u[1]);
}
else {
const int log_range_out = AOMMAX(16, bd + 6);
const __m128i clamp_lo_out = _mm_set1_epi32(-(1 << (log_range_out - 1)));
const __m128i clamp_hi_out = _mm_set1_epi32((1 << (log_range_out - 1)) - 1);
neg_shift_sse4_1(u[0], u[4], out + 0, out + 1, &clamp_lo_out, &clamp_hi_out,
out_shift);
neg_shift_sse4_1(u[6], u[2], out + 2, out + 3, &clamp_lo_out, &clamp_hi_out,
out_shift);
neg_shift_sse4_1(u[3], u[7], out + 4, out + 5, &clamp_lo_out, &clamp_hi_out,
out_shift);
neg_shift_sse4_1(u[5], u[1], out + 6, out + 7, &clamp_lo_out, &clamp_hi_out,
out_shift);
}
}
static void iadst8x8_new_sse4_1(__m128i *in, __m128i *out,
int bit,
int do_cols,
int bd,
int out_shift) {
const int32_t *cospi = cospi_arr(bit);
const __m128i cospi4 = _mm_set1_epi32(cospi[4]);
const __m128i cospi60 = _mm_set1_epi32(cospi[60]);
const __m128i cospi20 = _mm_set1_epi32(cospi[20]);
const __m128i cospi44 = _mm_set1_epi32(cospi[44]);
const __m128i cospi36 = _mm_set1_epi32(cospi[36]);
const __m128i cospi28 = _mm_set1_epi32(cospi[28]);
const __m128i cospi52 = _mm_set1_epi32(cospi[52]);
const __m128i cospi12 = _mm_set1_epi32(cospi[12]);
const __m128i cospi16 = _mm_set1_epi32(cospi[16]);
const __m128i cospi48 = _mm_set1_epi32(cospi[48]);
const __m128i cospim48 = _mm_set1_epi32(-cospi[48]);
const __m128i cospi32 = _mm_set1_epi32(cospi[32]);
const __m128i rnding = _mm_set1_epi32(1 << (bit - 1));
const __m128i kZero = _mm_setzero_si128();
const int log_range = AOMMAX(16, bd + (do_cols ? 6 : 8));
const __m128i clamp_lo = _mm_set1_epi32(-(1 << (log_range - 1)));
const __m128i clamp_hi = _mm_set1_epi32((1 << (log_range - 1)) - 1);
__m128i u[8], v[8], x;
// stage 0
// stage 1
// stage 2
u[0] = _mm_mullo_epi32(in[7], cospi4);
x = _mm_mullo_epi32(in[0], cospi60);
u[0] = _mm_add_epi32(u[0], x);
u[0] = _mm_add_epi32(u[0], rnding);
u[0] = _mm_srai_epi32(u[0], bit);
u[1] = _mm_mullo_epi32(in[7], cospi60);
x = _mm_mullo_epi32(in[0], cospi4);
u[1] = _mm_sub_epi32(u[1], x);
u[1] = _mm_add_epi32(u[1], rnding);
u[1] = _mm_srai_epi32(u[1], bit);
// (2)
u[2] = _mm_mullo_epi32(in[5], cospi20);
x = _mm_mullo_epi32(in[2], cospi44);
u[2] = _mm_add_epi32(u[2], x);
u[2] = _mm_add_epi32(u[2], rnding);
u[2] = _mm_srai_epi32(u[2], bit);
u[3] = _mm_mullo_epi32(in[5], cospi44);
x = _mm_mullo_epi32(in[2], cospi20);
u[3] = _mm_sub_epi32(u[3], x);
u[3] = _mm_add_epi32(u[3], rnding);
u[3] = _mm_srai_epi32(u[3], bit);
// (3)
u[4] = _mm_mullo_epi32(in[3], cospi36);
x = _mm_mullo_epi32(in[4], cospi28);
u[4] = _mm_add_epi32(u[4], x);
u[4] = _mm_add_epi32(u[4], rnding);
u[4] = _mm_srai_epi32(u[4], bit);
u[5] = _mm_mullo_epi32(in[3], cospi28);
x = _mm_mullo_epi32(in[4], cospi36);
u[5] = _mm_sub_epi32(u[5], x);
u[5] = _mm_add_epi32(u[5], rnding);
u[5] = _mm_srai_epi32(u[5], bit);
// (4)
u[6] = _mm_mullo_epi32(in[1], cospi52);
x = _mm_mullo_epi32(in[6], cospi12);
u[6] = _mm_add_epi32(u[6], x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_mullo_epi32(in[1], cospi12);
x = _mm_mullo_epi32(in[6], cospi52);
u[7] = _mm_sub_epi32(u[7], x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 3
addsub_sse4_1(u[0], u[4], &v[0], &v[4], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[1], u[5], &v[1], &v[5], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[2], u[6], &v[2], &v[6], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[3], u[7], &v[3], &v[7], &clamp_lo, &clamp_hi);
// stage 4
u[0] = v[0];
u[1] = v[1];
u[2] = v[2];
u[3] = v[3];
u[4] = _mm_mullo_epi32(v[4], cospi16);
x = _mm_mullo_epi32(v[5], cospi48);
u[4] = _mm_add_epi32(u[4], x);
u[4] = _mm_add_epi32(u[4], rnding);
u[4] = _mm_srai_epi32(u[4], bit);
u[5] = _mm_mullo_epi32(v[4], cospi48);
x = _mm_mullo_epi32(v[5], cospi16);
u[5] = _mm_sub_epi32(u[5], x);
u[5] = _mm_add_epi32(u[5], rnding);
u[5] = _mm_srai_epi32(u[5], bit);
u[6] = _mm_mullo_epi32(v[6], cospim48);
x = _mm_mullo_epi32(v[7], cospi16);
u[6] = _mm_add_epi32(u[6], x);
u[6] = _mm_add_epi32(u[6], rnding);
u[6] = _mm_srai_epi32(u[6], bit);
u[7] = _mm_mullo_epi32(v[6], cospi16);
x = _mm_mullo_epi32(v[7], cospim48);
u[7] = _mm_sub_epi32(u[7], x);
u[7] = _mm_add_epi32(u[7], rnding);
u[7] = _mm_srai_epi32(u[7], bit);
// stage 5
addsub_sse4_1(u[0], u[2], &v[0], &v[2], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[1], u[3], &v[1], &v[3], &clamp_lo, &clamp_hi);
addsub_sse4_1(u[4], u[6], &v[4], &v[6], &clamp_lo, &clamp_hi);
--> --------------------
--> maximum size reached
--> --------------------
