/*
* Copyright (c) 2020, Alliance for Open Media. All rights reserved.
*
* This source code is subject to the terms of the BSD 2 Clause License and
* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
* was not distributed with this source code in the LICENSE file, you can
* obtain it at www.aomedia.org/license/software. If the Alliance for Open
* Media Patent License 1.0 was not distributed with this source code in the
* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
*/
#include <arm_neon.h>
#include <assert.h>
#include "config/aom_config.h"
#include "config/av1_rtcd.h"
#include "aom/aom_integer.h"
#include "aom_dsp/arm/mem_neon.h"
#include "aom_dsp/arm/sum_neon.h"
#define MAX_UPSAMPLE_SZ
16
#define FILTER_INTRA_SCALE_BITS
4
// These kernels are a transposed version of those defined in reconintra.c,
// with the absolute value of the negatives taken in the top row.
DECLARE_ALIGNED(
16,
static const uint8_t,
av1_filter_intra_taps_neon[FILTER_INTRA_MODES][
7][
8]) = {
// clang-format off
{
{
6,
5,
3,
3,
4,
3,
3,
3 },
{
10,
2,
1,
1,
6,
2,
2,
1 },
{
0,
10,
1,
1,
0,
6,
2,
2 },
{
0,
0,
10,
2,
0,
0,
6,
2 },
{
0,
0,
0,
10,
0,
0,
0,
6 },
{
12,
9,
7,
5,
2,
2,
2,
3 },
{
0,
0,
0,
0,
12,
9,
7,
5 }
},
{
{
10,
6,
4,
2,
10,
6,
4,
2 },
{
16,
0,
0,
0,
16,
0,
0,
0 },
{
0,
16,
0,
0,
0,
16,
0,
0 },
{
0,
0,
16,
0,
0,
0,
16,
0 },
{
0,
0,
0,
16,
0,
0,
0,
16 },
{
10,
6,
4,
2,
0,
0,
0,
0 },
{
0,
0,
0,
0,
10,
6,
4,
2 }
},
{
{
8,
8,
8,
8,
4,
4,
4,
4 },
{
8,
0,
0,
0,
4,
0,
0,
0 },
{
0,
8,
0,
0,
0,
4,
0,
0 },
{
0,
0,
8,
0,
0,
0,
4,
0 },
{
0,
0,
0,
8,
0,
0,
0,
4 },
{
16,
16,
16,
16,
0,
0,
0,
0 },
{
0,
0,
0,
0,
16,
16,
16,
16 }
},
{
{
2,
1,
1,
0,
1,
1,
1,
1 },
{
8,
3,
2,
1,
4,
3,
2,
2 },
{
0,
8,
3,
2,
0,
4,
3,
2 },
{
0,
0,
8,
3,
0,
0,
4,
3 },
{
0,
0,
0,
8,
0,
0,
0,
4 },
{
10,
6,
4,
2,
3,
4,
4,
3 },
{
0,
0,
0,
0,
10,
6,
4,
3 }
},
{
{
12,
10,
9,
8,
10,
9,
8,
7 },
{
14,
0,
0,
0,
12,
1,
0,
0 },
{
0,
14,
0,
0,
0,
12,
0,
0 },
{
0,
0,
14,
0,
0,
0,
12,
1 },
{
0,
0,
0,
14,
0,
0,
0,
12 },
{
14,
12,
11,
10,
0,
0,
1,
1 },
{
0,
0,
0,
0,
14,
12,
11,
9 }
}
// clang-format on
};
static inline uint8x8_t filter_intra_predictor(
uint8x8_t s0, uint8x8_t s1, uint8x8_t s2, uint8x8_t s3, uint8x8_t s4,
uint8x8_t s5, uint8x8_t s6,
const uint8x8_t f0,
const uint8x8_t f1,
const uint8x8_t f2,
const uint8x8_t f3,
const uint8x8_t f4,
const uint8x8_t f5,
const uint8x8_t f6) {
uint16x8_t acc = vmull_u8(s1, f1);
// First row of each filter has all negative values so subtract.
acc = vmlsl_u8(acc, s0, f0);
acc = vmlal_u8(acc, s2, f2);
acc = vmlal_u8(acc, s3, f3);
acc = vmlal_u8(acc, s4, f4);
acc = vmlal_u8(acc, s5, f5);
acc = vmlal_u8(acc, s6, f6);
return vqrshrun_n_s16(vreinterpretq_s16_u16(acc), FILTER_INTRA_SCALE_BITS);
}
void av1_filter_intra_predictor_neon(uint8_t *dst, ptrdiff_t stride,
TX_SIZE tx_size,
const uint8_t *above,
const uint8_t *left, int mode) {
const int width = tx_size_wide[tx_size];
const int height = tx_size_high[tx_size];
assert(width <=
32 && height <=
32);
const uint8x8_t f0 = vld1_u8(av1_filter_intra_taps_neon[mode][
0]);
const uint8x8_t f1 = vld1_u8(av1_filter_intra_taps_neon[mode][
1]);
const uint8x8_t f2 = vld1_u8(av1_filter_intra_taps_neon[mode][
2]);
const uint8x8_t f3 = vld1_u8(av1_filter_intra_taps_neon[mode][
3]);
const uint8x8_t f4 = vld1_u8(av1_filter_intra_taps_neon[mode][
4]);
const uint8x8_t f5 = vld1_u8(av1_filter_intra_taps_neon[mode][
5]);
const uint8x8_t f6 = vld1_u8(av1_filter_intra_taps_neon[mode][
6]);
// Computing 4 cols per iteration (instead of 8) for 8x<h> blocks is faster.
if (width <=
8) {
uint8x8_t s0 = vdup_n_u8(above[-
1]);
uint8x8_t s5 = vdup_n_u8(left[
0]);
uint8x8_t s6 = vdup_n_u8(left[
1]);
int c =
0;
do {
uint8x8_t s1234 = load_unaligned_u8_4x1(above + c);
uint8x8_t s1 = vdup_lane_u8(s1234,
0);
uint8x8_t s2 = vdup_lane_u8(s1234,
1);
uint8x8_t s3 = vdup_lane_u8(s1234,
2);
uint8x8_t s4 = vdup_lane_u8(s1234,
3);
uint8x8_t res = filter_intra_predictor(s0, s1, s2, s3, s4, s5, s6, f0, f1,
f2, f3, f4, f5, f6);
store_u8x4_strided_x2(dst + c, stride, res);
s0 = s4;
s5 = vdup_lane_u8(res,
3);
s6 = vdup_lane_u8(res,
7);
c +=
4;
}
while (c < width);
int r =
2;
do {
s0 = vdup_n_u8(left[r -
1]);
s5 = vdup_n_u8(left[r +
0]);
s6 = vdup_n_u8(left[r +
1]);
c =
0;
do {
uint8x8_t s1234 = load_u8_4x1(dst + (r -
1) * stride + c);
uint8x8_t s1 = vdup_lane_u8(s1234,
0);
uint8x8_t s2 = vdup_lane_u8(s1234,
1);
uint8x8_t s3 = vdup_lane_u8(s1234,
2);
uint8x8_t s4 = vdup_lane_u8(s1234,
3);
uint8x8_t res = filter_intra_predictor(s0, s1, s2, s3, s4, s5, s6, f0,
f1, f2, f3, f4, f5, f6);
store_u8x4_strided_x2(dst + r * stride + c, stride, res);
s0 = s4;
s5 = vdup_lane_u8(res,
3);
s6 = vdup_lane_u8(res,
7);
c +=
4;
}
while (c < width);
r +=
2;
}
while (r < height);
}
else {
uint8x8_t s0_lo = vdup_n_u8(above[-
1]);
uint8x8_t s5_lo = vdup_n_u8(left[
0]);
uint8x8_t s6_lo = vdup_n_u8(left[
1]);
int c =
0;
do {
uint8x8_t s1234 = vld1_u8(above + c);
uint8x8_t s1_lo = vdup_lane_u8(s1234,
0);
uint8x8_t s2_lo = vdup_lane_u8(s1234,
1);
uint8x8_t s3_lo = vdup_lane_u8(s1234,
2);
uint8x8_t s4_lo = vdup_lane_u8(s1234,
3);
uint8x8_t res_lo =
filter_intra_predictor(s0_lo, s1_lo, s2_lo, s3_lo, s4_lo, s5_lo,
s6_lo, f0, f1, f2, f3, f4, f5, f6);
uint8x8_t s0_hi = s4_lo;
uint8x8_t s1_hi = vdup_lane_u8(s1234,
4);
uint8x8_t s2_hi = vdup_lane_u8(s1234,
5);
uint8x8_t s3_hi = vdup_lane_u8(s1234,
6);
uint8x8_t s4_hi = vdup_lane_u8(s1234,
7);
uint8x8_t s5_hi = vdup_lane_u8(res_lo,
3);
uint8x8_t s6_hi = vdup_lane_u8(res_lo,
7);
uint8x8_t res_hi =
filter_intra_predictor(s0_hi, s1_hi, s2_hi, s3_hi, s4_hi, s5_hi,
s6_hi, f0, f1, f2, f3, f4, f5, f6);
uint32x2x2_t res =
vzip_u32(vreinterpret_u32_u8(res_lo), vreinterpret_u32_u8(res_hi));
vst1_u8(dst +
0 * stride + c, vreinterpret_u8_u32(res.val[
0]));
vst1_u8(dst +
1 * stride + c, vreinterpret_u8_u32(res.val[
1]));
s0_lo = s4_hi;
s5_lo = vdup_lane_u8(res_hi,
3);
s6_lo = vdup_lane_u8(res_hi,
7);
c +=
8;
}
while (c < width);
int r =
2;
do {
s0_lo = vdup_n_u8(left[r -
1]);
s5_lo = vdup_n_u8(left[r +
0]);
s6_lo = vdup_n_u8(left[r +
1]);
c =
0;
do {
uint8x8_t s1234 = vld1_u8(dst + (r -
1) * stride + c);
uint8x8_t s1_lo = vdup_lane_u8(s1234,
0);
uint8x8_t s2_lo = vdup_lane_u8(s1234,
1);
uint8x8_t s3_lo = vdup_lane_u8(s1234,
2);
uint8x8_t s4_lo = vdup_lane_u8(s1234,
3);
uint8x8_t res_lo =
filter_intra_predictor(s0_lo, s1_lo, s2_lo, s3_lo, s4_lo, s5_lo,
s6_lo, f0, f1, f2, f3, f4, f5, f6);
uint8x8_t s0_hi = s4_lo;
uint8x8_t s1_hi = vdup_lane_u8(s1234,
4);
uint8x8_t s2_hi = vdup_lane_u8(s1234,
5);
uint8x8_t s3_hi = vdup_lane_u8(s1234,
6);
uint8x8_t s4_hi = vdup_lane_u8(s1234,
7);
uint8x8_t s5_hi = vdup_lane_u8(res_lo,
3);
uint8x8_t s6_hi = vdup_lane_u8(res_lo,
7);
uint8x8_t res_hi =
filter_intra_predictor(s0_hi, s1_hi, s2_hi, s3_hi, s4_hi, s5_hi,
s6_hi, f0, f1, f2, f3, f4, f5, f6);
uint32x2x2_t res =
vzip_u32(vreinterpret_u32_u8(res_lo), vreinterpret_u32_u8(res_hi));
vst1_u8(dst + (r +
0) * stride + c, vreinterpret_u8_u32(res.val[
0]));
vst1_u8(dst + (r +
1) * stride + c, vreinterpret_u8_u32(res.val[
1]));
s0_lo = s4_hi;
s5_lo = vdup_lane_u8(res_hi,
3);
s6_lo = vdup_lane_u8(res_hi,
7);
c +=
8;
}
while (c < width);
r +=
2;
}
while (r < height);
}
}
void av1_filter_intra_edge_neon(uint8_t *p, int sz, int strength) {
if (!strength)
return;
assert(sz >=
0 && sz <=
129);
uint8_t edge[
160];
// Max value of sz + enough padding for vector accesses.
memcpy(edge +
1, p, sz *
sizeof(*p));
// Populate extra space appropriately.
edge[
0] = edge[
1];
edge[sz +
1] = edge[sz];
edge[sz +
2] = edge[sz];
// Don't overwrite first pixel.
uint8_t *dst = p +
1;
sz--;
if (strength ==
1) {
// Filter: {4, 8, 4}.
const uint8_t *src = edge +
1;
while (sz >=
8) {
uint8x8_t s0 = vld1_u8(src);
uint8x8_t s1 = vld1_u8(src +
1);
uint8x8_t s2 = vld1_u8(src +
2);
// Make use of the identity:
// (4*a + 8*b + 4*c) >> 4 == (a + (b << 1) + c) >> 2
uint16x8_t t0 = vaddl_u8(s0, s2);
uint16x8_t t1 = vaddl_u8(s1, s1);
uint16x8_t sum = vaddq_u16(t0, t1);
uint8x8_t res = vrshrn_n_u16(sum,
2);
vst1_u8(dst, res);
src +=
8;
dst +=
8;
sz -=
8;
}
if (sz >
0) {
// Handle sz < 8 to avoid modifying out-of-bounds values.
uint8x8_t s0 = vld1_u8(src);
uint8x8_t s1 = vld1_u8(src +
1);
uint8x8_t s2 = vld1_u8(src +
2);
uint16x8_t t0 = vaddl_u8(s0, s2);
uint16x8_t t1 = vaddl_u8(s1, s1);
uint16x8_t sum = vaddq_u16(t0, t1);
uint8x8_t res = vrshrn_n_u16(sum,
2);
// Mask off out-of-bounds indices.
uint8x8_t current_dst = vld1_u8(dst);
uint8x8_t mask = vcgt_u8(vdup_n_u8(sz), vcreate_u8(
0x0706050403020100));
res = vbsl_u8(mask, res, current_dst);
vst1_u8(dst, res);
}
}
else if (strength ==
2) {
// Filter: {5, 6, 5}.
const uint8_t *src = edge +
1;
const uint8x8x3_t filter = { { vdup_n_u8(
5), vdup_n_u8(
6), vdup_n_u8(
5) } };
while (sz >=
8) {
uint8x8_t s0 = vld1_u8(src);
uint8x8_t s1 = vld1_u8(src +
1);
uint8x8_t s2 = vld1_u8(src +
2);
uint16x8_t accum = vmull_u8(s0, filter.val[
0]);
accum = vmlal_u8(accum, s1, filter.val[
1]);
accum = vmlal_u8(accum, s2, filter.val[
2]);
uint8x8_t res = vrshrn_n_u16(accum,
4);
vst1_u8(dst, res);
src +=
8;
dst +=
8;
sz -=
8;
}
if (sz >
0) {
// Handle sz < 8 to avoid modifying out-of-bounds values.
uint8x8_t s0 = vld1_u8(src);
uint8x8_t s1 = vld1_u8(src +
1);
uint8x8_t s2 = vld1_u8(src +
2);
uint16x8_t accum = vmull_u8(s0, filter.val[
0]);
accum = vmlal_u8(accum, s1, filter.val[
1]);
accum = vmlal_u8(accum, s2, filter.val[
2]);
uint8x8_t res = vrshrn_n_u16(accum,
4);
// Mask off out-of-bounds indices.
uint8x8_t current_dst = vld1_u8(dst);
uint8x8_t mask = vcgt_u8(vdup_n_u8(sz), vcreate_u8(
0x0706050403020100));
res = vbsl_u8(mask, res, current_dst);
vst1_u8(dst, res);
}
}
else {
// Filter {2, 4, 4, 4, 2}.
const uint8_t *src = edge;
while (sz >=
8) {
uint8x8_t s0 = vld1_u8(src);
uint8x8_t s1 = vld1_u8(src +
1);
uint8x8_t s2 = vld1_u8(src +
2);
uint8x8_t s3 = vld1_u8(src +
3);
uint8x8_t s4 = vld1_u8(src +
4);
// Make use of the identity:
// (2*a + 4*b + 4*c + 4*d + 2*e) >> 4 == (a + ((b + c + d) << 1) + e) >> 3
uint16x8_t t0 = vaddl_u8(s0, s4);
uint16x8_t t1 = vaddl_u8(s1, s2);
t1 = vaddw_u8(t1, s3);
t1 = vaddq_u16(t1, t1);
uint16x8_t sum = vaddq_u16(t0, t1);
uint8x8_t res = vrshrn_n_u16(sum,
3);
vst1_u8(dst, res);
src +=
8;
dst +=
8;
sz -=
8;
}
if (sz >
0) {
// Handle sz < 8 to avoid modifying out-of-bounds values.
uint8x8_t s0 = vld1_u8(src);
uint8x8_t s1 = vld1_u8(src +
1);
uint8x8_t s2 = vld1_u8(src +
2);
uint8x8_t s3 = vld1_u8(src +
3);
uint8x8_t s4 = vld1_u8(src +
4);
uint16x8_t t0 = vaddl_u8(s0, s4);
uint16x8_t t1 = vaddl_u8(s1, s2);
t1 = vaddw_u8(t1, s3);
t1 = vaddq_u16(t1, t1);
uint16x8_t sum = vaddq_u16(t0, t1);
uint8x8_t res = vrshrn_n_u16(sum,
3);
// Mask off out-of-bounds indices.
uint8x8_t current_dst = vld1_u8(dst);
uint8x8_t mask = vcgt_u8(vdup_n_u8(sz), vcreate_u8(
0x0706050403020100));
res = vbsl_u8(mask, res, current_dst);
vst1_u8(dst, res);
}
}
}
void av1_upsample_intra_edge_neon(uint8_t *p, int sz) {
if (!sz)
return;
assert(sz <= MAX_UPSAMPLE_SZ);
uint8_t edge[MAX_UPSAMPLE_SZ +
3];
const uint8_t *src = edge;
// Copy p[-1..(sz-1)] and pad out both ends.
edge[
0] = p[-
1];
edge[
1] = p[-
1];
memcpy(edge +
2, p, sz);
edge[sz +
2] = p[sz -
1];
p[-
2] = p[-
1];
uint8_t *dst = p -
1;
do {
uint8x8_t s0 = vld1_u8(src);
uint8x8_t s1 = vld1_u8(src +
1);
uint8x8_t s2 = vld1_u8(src +
2);
uint8x8_t s3 = vld1_u8(src +
3);
int16x8_t t0 = vreinterpretq_s16_u16(vaddl_u8(s0, s3));
int16x8_t t1 = vreinterpretq_s16_u16(vaddl_u8(s1, s2));
t1 = vmulq_n_s16(t1,
9);
t1 = vsubq_s16(t1, t0);
uint8x8x2_t res = { { vqrshrun_n_s16(t1,
4), s2 } };
vst2_u8(dst, res);
src +=
8;
dst +=
16;
sz -=
8;
}
while (sz >
0);
}
