Quelle  ipred.S   Sprache: Sparc

/*
 * Copyright © 2018, VideoLAN and dav1d authors
 * Copyright © 2020, Martin Storsjo
 * Copyright © 2019, B Krishnan Iyer
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "src/arm/asm.S"
#include "util.S"

// void ipred_dc_128_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                             const pixel *const topleft,
//                             const int width, const int height, const int a,
//                             const int max_width, const int max_height);
function ipred_dc_128_8bpc_neon, export=1
        push            {r4, lr}
        ldr             r4,  [sp, #8]
        clz             r3,  r3
        adr             r2,  L(ipred_dc_128_tbl)
        sub             r3,  r3,  #25
        ldr             r3,  [r2,  r3,  lsl #2]
        vmov.i8         q0,  #128
        add             r2,  r2,  r3
        add             r12, r0,  r1
        lsl             r1,  r1,  #1
        bx              r2

        .align 2
L(ipred_dc_128_tbl):
        .word 640f - L(ipred_dc_128_tbl) + CONFIG_THUMB
        .word 320f - L(ipred_dc_128_tbl) + CONFIG_THUMB
        .word 16f  - L(ipred_dc_128_tbl) + CONFIG_THUMB
        .word 8f   - L(ipred_dc_128_tbl) + CONFIG_THUMB
        .word 4f   - L(ipred_dc_128_tbl) + CONFIG_THUMB
4:
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        subs            r4,  r4,  #4
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        bgt             4b
        pop             {r4, pc}
8:
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        subs            r4,  r4,  #4
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        bgt             8b
        pop             {r4, pc}
16:
        vst1.8          {d0,  d1}, [r0,  :128], r1
        vst1.8          {d0,  d1}, [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1}, [r0,  :128], r1
        vst1.8          {d0,  d1}, [r12, :128], r1
        bgt             16b
        pop             {r4, pc}
320:
        vmov.i8         q1,  #128
32:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             32b
        pop             {r4, pc}
640:
        vmov.i8         q1,  #128
        sub             r1,  r1,  #32
64:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             64b
        pop             {r4, pc}
endfunc

// void ipred_v_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                        const pixel *const topleft,
//                        const int width, const int height, const int a,
//                        const int max_width, const int max_height);
function ipred_v_8bpc_neon, export=1
        push            {r4, lr}
        ldr             lr,  [sp, #8]
        clz             r3,  r3
        adr             r4,  L(ipred_v_tbl)
        sub             r3,  r3,  #25
        ldr             r3,  [r4,  r3,  lsl #2]
        add             r2,  r2,  #1
        add             r4,  r4,  r3
        add             r12, r0,  r1
        lsl             r1,  r1,  #1
        bx              r4

        .align 2
L(ipred_v_tbl):
        .word 640f - L(ipred_v_tbl) + CONFIG_THUMB
        .word 320f - L(ipred_v_tbl) + CONFIG_THUMB
        .word 160f - L(ipred_v_tbl) + CONFIG_THUMB
        .word 80f  - L(ipred_v_tbl) + CONFIG_THUMB
        .word 40f  - L(ipred_v_tbl) + CONFIG_THUMB
40:
        vld1.32         {d0[]},   [r2]
4:
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        subs            lr,  lr,  #4
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        bgt             4b
        pop             {r4, pc}
80:
        vld1.8          {d0}, [r2]
8:
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        subs            lr,  lr,  #4
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        bgt             8b
        pop             {r4, pc}
160:
        vld1.8          {q0},  [r2]
16:
        vst1.8          {d0,  d1},  [r0,  :128], r1
        vst1.8          {d0,  d1},  [r12, :128], r1
        subs            lr,  lr,  #4
        vst1.8          {d0,  d1},  [r0,  :128], r1
        vst1.8          {d0,  d1},  [r12, :128], r1
        bgt             16b
        pop             {r4, pc}
320:
        vld1.8          {q0,  q1},  [r2]
32:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            lr,  lr,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             32b
        pop             {r4, pc}
640:
        vld1.8          {q0,  q1},  [r2]!
        sub             r1,  r1,  #32
        vld1.8          {q2,  q3},  [r2]
64:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d4,  d5,  d6,  d7},  [r0,  :128], r1
        vst1.8          {d4,  d5,  d6,  d7},  [r12, :128], r1
        subs            lr,  lr,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d4,  d5,  d6,  d7},  [r0,  :128], r1
        vst1.8          {d4,  d5,  d6,  d7},  [r12, :128], r1
        bgt             64b
        pop             {r4, pc}
endfunc

// void ipred_h_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                        const pixel *const topleft,
//                        const int width, const int height, const int a,
//                        const int max_width, const int max_height);
function ipred_h_8bpc_neon, export=1
        push            {r4-r5, lr}
        ldr             r4,  [sp, #12]
        clz             r3,  r3
        adr             r5,  L(ipred_h_tbl)
        sub             r3,  r3,  #25
        ldr             r3,  [r5,  r3,  lsl #2]
        sub             r2,  r2,  #4
        mov             lr,  #-4
        add             r5,  r5,  r3
        add             r12, r0,  r1
        lsl             r1,  r1,  #1
        bx              r5

        .align 2
L(ipred_h_tbl):
        .word 640f - L(ipred_h_tbl) + CONFIG_THUMB
        .word 320f - L(ipred_h_tbl) + CONFIG_THUMB
        .word 160f - L(ipred_h_tbl) + CONFIG_THUMB
        .word 8f   - L(ipred_h_tbl) + CONFIG_THUMB
        .word 4f   - L(ipred_h_tbl) + CONFIG_THUMB
4:
        vld4.8          {d0[],  d1[],  d2[],  d3[]},  [r2, :32],  lr
        vst1.32         {d3[0]},  [r0,  :32], r1
        vst1.32         {d2[0]},  [r12, :32], r1
        subs            r4,  r4,  #4
        vst1.32         {d1[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        bgt             4b
        pop             {r4-r5, pc}
8:
        vld4.8          {d0[],  d1[],  d2[],  d3[]},  [r2, :32],  lr
        vst1.8          {d3},  [r0,  :64], r1
        vst1.8          {d2},  [r12, :64], r1
        subs            r4,  r4,  #4
        vst1.8          {d1},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        bgt             8b
        pop             {r4-r5, pc}
160:
        add             r2,  r2,  #3
        mov             lr,  #-1
16:
        vld1.8          {d0[],  d1[]},  [r2],  lr
        subs            r4,  r4,  #4
        vld1.8          {d2[],  d3[]},  [r2],  lr
        vst1.8          {q0},  [r0,    :128],  r1
        vld1.8          {d4[],  d5[]},  [r2],  lr
        vst1.8          {q1},  [r12,   :128],  r1
        vld1.8          {d6[],  d7[]},  [r2],  lr
        vst1.8          {q2},  [r0,    :128],  r1
        vst1.8          {q3},  [r12,   :128],  r1
        bgt             16b
        pop             {r4-r5, pc}
320:
        add             r2,  r2,  #3
        mov             lr,  #-1
        sub             r1,  r1,  #16
32:
        vld1.8          {d0[],  d1[]}, [r2],  lr
        subs            r4,  r4,  #4
        vld1.8          {d2[],  d3[]}, [r2],  lr
        vst1.8          {q0},  [r0,   :128]!
        vld1.8          {d4[],  d5[]}, [r2],  lr
        vst1.8          {q1},  [r12,  :128]!
        vld1.8          {d6[],  d7[]}, [r2],  lr
        vst1.8          {q0},  [r0,   :128],  r1
        vst1.8          {q1},  [r12,  :128],  r1
        vst1.8          {q2},  [r0,   :128]!
        vst1.8          {q3},  [r12,  :128]!
        vst1.8          {q2},  [r0,   :128],  r1
        vst1.8          {q3},  [r12,  :128],  r1
        bgt             32b
        pop             {r4-r5, pc}
640:
        add             r2,  r2,  #3
        mov             lr,  #-1
        sub             r1,  r1,  #48
64:
        vld1.8          {d0[],  d1[]},  [r2],  lr
        subs            r4,  r4,  #4
        vld1.8          {d2[],  d3[]},  [r2],  lr
        vst1.8          {q0},  [r0,    :128]!
        vld1.8          {d4[],  d5[]},  [r2],  lr
        vst1.8          {q1},  [r12,   :128]!
        vld1.8          {d6[],  d7[]},  [r2],  lr
        vst1.8          {q0},  [r0,    :128]!
        vst1.8          {q1},  [r12,   :128]!
        vst1.8          {q0},  [r0,    :128]!
        vst1.8          {q1},  [r12,   :128]!
        vst1.8          {q0},  [r0,    :128],  r1
        vst1.8          {q1},  [r12,   :128],  r1
        vst1.8          {q2},  [r0,    :128]!
        vst1.8          {q3},  [r12,   :128]!
        vst1.8          {q2},  [r0,    :128]!
        vst1.8          {q3},  [r12,   :128]!
        vst1.8          {q2},  [r0,    :128]!
        vst1.8          {q3},  [r12,   :128]!
        vst1.8          {q2},  [r0,    :128],  r1
        vst1.8          {q3},  [r12,   :128],  r1
        bgt             64b
        pop             {r4-r5, pc}
endfunc

// void ipred_dc_top_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                             const pixel *const topleft,
//                             const int width, const int height, const int a,
//                             const int max_width, const int max_height);
function ipred_dc_top_8bpc_neon, export=1
        push            {r4-r5, lr}
        ldr             r4,  [sp, #12]
        clz             r3,  r3
        adr             r5,  L(ipred_dc_top_tbl)
        sub             r3,  r3,  #25
        ldr             r3,  [r5,  r3,  lsl #2]
        add             r2,  r2,  #1
        add             r5,  r5,  r3
        add             r12, r0,  r1
        lsl             r1,  r1,  #1
        bx              r5

        .align 2
L(ipred_dc_top_tbl):
        .word 640f - L(ipred_dc_top_tbl) + CONFIG_THUMB
        .word 320f - L(ipred_dc_top_tbl) + CONFIG_THUMB
        .word 160f - L(ipred_dc_top_tbl) + CONFIG_THUMB
        .word 80f  - L(ipred_dc_top_tbl) + CONFIG_THUMB
        .word 40f  - L(ipred_dc_top_tbl) + CONFIG_THUMB
40:
        vld1.32         {d0[]},  [r2]
        vpaddl.u8       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d0,  q0,  #2
        vdup.8          d0,  d0[0]
4:
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        subs            r4,  r4,  #4
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        bgt             4b
        pop             {r4-r5, pc}
80:
        vld1.8          {d0},  [r2]
        vpaddl.u8       d0,  d0
        vpadd.u16       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d0,  q0,  #3
        vdup.8          d0,  d0[0]
8:
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        subs            r4,  r4,  #4
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        bgt             8b
        pop             {r4-r5, pc}
160:
        vld1.8          {d0,  d1},  [r2]
        vaddl.u8        q0,  d0,  d1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d0,  q0,  #4
        vdup.8          q0,  d0[0]
16:
        vst1.8          {d0,  d1},  [r0,  :128], r1
        vst1.8          {d0,  d1},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1},  [r0,  :128], r1
        vst1.8          {d0,  d1},  [r12, :128], r1
        bgt             16b
        pop             {r4-r5, pc}
320:
        vld1.8          {d0,  d1,  d2,  d3},  [r2]
        vaddl.u8        q0,  d0,  d1
        vaddl.u8        q1,  d2,  d3
        vadd.u16        q0,  q0,  q1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d4,  q0,  #5
        vdup.8          q0,  d4[0]
        vdup.8          q1,  d4[0]
32:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             32b
        pop             {r4-r5, pc}
640:
        vld1.8          {d0,  d1,  d2,  d3},  [r2]!
        vaddl.u8        q0,  d0,  d1
        vld1.8          {d4,  d5,  d6,  d7},  [r2]
        vaddl.u8        q1,  d2,  d3
        vaddl.u8        q2,  d4,  d5
        vaddl.u8        q3,  d6,  d7
        vadd.u16        q0,  q0,  q1
        vadd.u16        q1,  q2,  q3
        vadd.u16        q0,  q0,  q1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d18, q0,  #6
        vdup.8          q0,  d18[0]
        vdup.8          q1,  d18[0]
        sub             r1,  r1,  #32
64:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             64b
        pop             {r4-r5, pc}
endfunc

// void ipred_dc_left_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                              const pixel *const topleft,
//                              const int width, const int height, const int a,
//                              const int max_width, const int max_height);
function ipred_dc_left_8bpc_neon, export=1
        push            {r4-r5, lr}
        ldr             r4,  [sp, #12]
        sub             r2,  r2,  r4
        clz             r3,  r3
        clz             lr,  r4
        sub             lr,  lr,  #25
        adr             r5,  L(ipred_dc_left_tbl)
        sub             r3,  r3,  #20
        ldr             r3,  [r5,  r3,  lsl #2]
        ldr             lr,  [r5,  lr,  lsl #2]
        add             r3,  r5,  r3
        add             r5,  r5,  lr
        add             r12, r0,  r1
        lsl             r1,  r1,  #1
        bx              r5

        .align 2
L(ipred_dc_left_tbl):
        .word L(ipred_dc_left_h64) - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_h32) - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_h16) - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_h8)  - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_h4)  - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_w64) - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_w32) - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_w16) - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_w8)  - L(ipred_dc_left_tbl) + CONFIG_THUMB
        .word L(ipred_dc_left_w4)  - L(ipred_dc_left_tbl) + CONFIG_THUMB

L(ipred_dc_left_h4):
        vld1.32         {d0[]},  [r2, :32]
        vpaddl.u8       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d0,  q0,  #2
        vdup.8          q0,  d0[0]
        bx              r3
L(ipred_dc_left_w4):
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        subs            r4,  r4,  #4
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        bgt             L(ipred_dc_left_w4)
        pop             {r4-r5, pc}
L(ipred_dc_left_h8):
        vld1.8          {d0},  [r2, :64]
        vpaddl.u8       d0,  d0
        vpadd.u16       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d0,  q0,  #3
        vdup.8          q0,  d0[0]
        bx              r3
L(ipred_dc_left_w8):
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        subs            r4,  r4,  #4
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        bgt             L(ipred_dc_left_w8)
        pop             {r4-r5, pc}
L(ipred_dc_left_h16):
        vld1.8          {d0,  d1},  [r2, :128]
        vaddl.u8        q0,  d0,  d1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d0,  q0,  #4
        vdup.8          q0,  d0[0]
        bx              r3
L(ipred_dc_left_w16):
        vst1.8          {d0,  d1},  [r0,  :128], r1
        vst1.8          {d0,  d1},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1},  [r0,  :128], r1
        vst1.8          {d0,  d1},  [r12, :128], r1
        bgt             L(ipred_dc_left_w16)
        pop             {r4-r5, pc}
L(ipred_dc_left_h32):
        vld1.8          {d0,  d1,  d2,  d3},  [r2, :128]
        vaddl.u8        q0,  d0,  d1
        vaddl.u8        q1,  d2,  d3
        vadd.u16        q0,  q0,  q1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d0,  q0,  #5
        vdup.8          q0,  d0[0]
        bx              r3
L(ipred_dc_left_w32):
        vmov.8          q1,  q0
1:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             1b
        pop             {r4-r5, pc}
L(ipred_dc_left_h64):
        vld1.8          {d0,  d1,  d2,  d3},  [r2, :128]!
        vld1.8          {d4,  d5,  d6,  d7},  [r2, :128]
        vaddl.u8        q0,  d0,  d1
        vaddl.u8        q1,  d2,  d3
        vaddl.u8        q2,  d4,  d5
        vaddl.u8        q3,  d6,  d7
        vadd.u16        q0,  q0,  q1
        vadd.u16        q1,  q2,  q3
        vadd.u16        q0,  q0,  q1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        vpadd.u16       d0,  d0
        vrshrn.u16      d0,  q0,  #6
        vdup.8          q0,  d0[0]
        bx              r3
L(ipred_dc_left_w64):
        vmov.8          q1,  q0
        sub             r1,  r1,  #32
1:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            r4,  r4, #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             1b
        pop             {r4-r5, pc}
endfunc

// void ipred_dc_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                         const pixel *const topleft,
//                         const int width, const int height, const int a,
//                         const int max_width, const int max_height);
function ipred_dc_8bpc_neon, export=1
        push            {r4-r6, lr}
        ldr             r4,  [sp, #16]
        sub             r2,  r2,  r4
        add             lr,  r3,  r4        // width + height
        clz             r3,  r3
        clz             r12, r4
        vdup.16         q15, lr             // width + height
        adr             r5,  L(ipred_dc_tbl)
        rbit            lr,  lr             // rbit(width + height)
        sub             r3,  r3,  #20       // 25 leading bits, minus table offset 5
        sub             r12, r12, #25
        clz             lr,  lr             // ctz(width + height)
        ldr             r3,  [r5,  r3,  lsl #2]
        ldr             r12, [r5,  r12, lsl #2]
        neg             lr,  lr             // -ctz(width + height)
        add             r3,  r5,  r3
        add             r5,  r5,  r12
        vshr.u16        q15, q15, #1        // (width + height) >> 1
        vdup.16         q14, lr             // -ctz(width + height)
        add             r12, r0,  r1
        lsl             r1,  r1,  #1
        bx              r5

        .align 2
L(ipred_dc_tbl):
        .word L(ipred_dc_h64) - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_h32) - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_h16) - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_h8)  - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_h4)  - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_w64) - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_w32) - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_w16) - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_w8)  - L(ipred_dc_tbl) + CONFIG_THUMB
        .word L(ipred_dc_w4)  - L(ipred_dc_tbl) + CONFIG_THUMB

L(ipred_dc_h4):
        vld1.32         {d0[]},  [r2, :32]!
        vpaddl.u8       d0,  d0
        add             r2,  r2,  #1
        vpadd.u16       d0,  d0
        bx              r3
L(ipred_dc_w4):
        vld1.32         {d1[]},  [r2]
        vadd.s16        d0,  d0,  d30
        vpaddl.u8       d1,  d1
        vpadd.u16       d1,  d1
        cmp             r4,  #4
        vadd.s16        d0,  d0,  d1
        vshl.u16        d0,  d0,  d28
        beq             1f
        // h = 8/16
        movw            lr,  #(0x3334/2)
        movw            r5,  #(0x5556/2)
        cmp             r4,  #16
        it              ne
        movne           lr,  r5
        vdup.16         d30, lr
        vqdmulh.s16     d0,  d0,  d30
1:
        vdup.8          d0,  d0[0]
2:
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        subs            r4,  r4,  #4
        vst1.32         {d0[0]},  [r0,  :32], r1
        vst1.32         {d0[0]},  [r12, :32], r1
        bgt             2b
        pop             {r4-r6, pc}

L(ipred_dc_h8):
        vld1.8          {d0},  [r2, :64]!
        vpaddl.u8       d0,  d0
        vpadd.u16       d0,  d0
        add             r2,  r2,  #1
        vpadd.u16       d0,  d0
        bx              r3
L(ipred_dc_w8):
        vld1.8          {d2},  [r2]
        vadd.s16        d0,  d0,  d30
        vpaddl.u8       d2,  d2
        vpadd.u16       d2,  d2
        vpadd.u16       d2,  d2
        cmp             r4,  #8
        vadd.s16        d0,  d0,  d2
        vshl.u16        d0,  d0,  d28
        beq             1f
        // h = 4/16/32
        cmp             r4,  #32
        movw            lr,  #(0x3334/2)
        movw            r5,  #(0x5556/2)
        it              ne
        movne           lr,  r5
        vdup.16         d24, lr
        vqdmulh.s16     d0,  d0,  d24
1:
        vdup.8          d0,  d0[0]
2:
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        subs            r4,  r4,  #4
        vst1.8          {d0},  [r0,  :64], r1
        vst1.8          {d0},  [r12, :64], r1
        bgt             2b
        pop             {r4-r6, pc}

L(ipred_dc_h16):
        vld1.8          {d0,  d1},  [r2, :128]!
        vaddl.u8        q0,  d0,  d1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        add             r2,  r2,  #1
        vpadd.u16       d0,  d0
        bx              r3
L(ipred_dc_w16):
        vld1.8          {d2,  d3},  [r2]
        vadd.s16        d0,  d0,  d30
        vaddl.u8        q1,  d2,  d3
        vadd.u16        d2,  d2,  d3
        vpadd.u16       d2,  d2
        vpadd.u16       d2,  d2
        cmp             r4,  #16
        vadd.s16        d0,  d0,  d2
        vshl.u16        d0,  d0,  d28
        beq             1f
        // h = 4/8/32/64
        tst             r4,  #(32+16+8)     // 16 added to make a consecutive bitmask
        movw            lr,  #(0x3334/2)
        movw            r5,  #(0x5556/2)
        it              ne
        movne           lr,  r5
        vdup.16         d24, lr
        vqdmulh.s16     d0,  d0,  d24
1:
        vdup.8          q0,  d0[0]
2:
        vst1.8          {d0,  d1},  [r0,  :128], r1
        vst1.8          {d0,  d1},  [r12, :128], r1
        subs            r4,  r4, #4
        vst1.8          {d0,  d1},  [r0,  :128], r1
        vst1.8          {d0,  d1},  [r12, :128], r1
        bgt             2b
        pop             {r4-r6, pc}

L(ipred_dc_h32):
        vld1.8          {d0,  d1,  d2,  d3},  [r2, :128]!
        vaddl.u8        q0,  d0,  d1
        vaddl.u8        q1,  d2,  d3
        vadd.u16        q0,  q0,  q1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        add             r2,  r2,  #1
        vpadd.u16       d0,  d0
        bx              r3
L(ipred_dc_w32):
        vld1.8          {d2,  d3,  d4,  d5},  [r2]
        vadd.s16        d0,  d0,  d30
        vaddl.u8        q1,  d2,  d3
        vaddl.u8        q2,  d4,  d5
        vadd.u16        q1,  q1,  q2
        vadd.u16        d2,  d2,  d3
        vpadd.u16       d2,  d2
        vpadd.u16       d2,  d2
        cmp             r4,  #32
        vadd.s16        d0,  d0,  d2
        vshl.u16        d4,  d0,  d28
        beq             1f
        // h = 8/16/64
        cmp             r4,  #8
        movw            lr,  #(0x3334/2)
        movw            r5,  #(0x5556/2)
        it              ne
        movne           lr,  r5
        vdup.16         d24, lr
        vqdmulh.s16     d4,  d4,  d24
1:
        vdup.8          q0,  d4[0]
        vdup.8          q1,  d4[0]
2:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             2b
        pop             {r4-r6, pc}

L(ipred_dc_h64):
        vld1.8          {d0,  d1,  d2,  d3},  [r2, :128]!
        vaddl.u8        q0,  d0,  d1
        vld1.8          {d4,  d5,  d6,  d7},  [r2, :128]!
        vaddl.u8        q1,  d2,  d3
        vaddl.u8        q2,  d4,  d5
        vaddl.u8        q3,  d6,  d7
        vadd.u16        q0,  q0,  q1
        vadd.u16        q1,  q2,  q3
        vadd.u16        q0,  q0,  q1
        vadd.u16        d0,  d0,  d1
        vpadd.u16       d0,  d0
        add             r2,  r2,  #1
        vpadd.u16       d0,  d0
        bx              r3
L(ipred_dc_w64):
        vld1.8          {d2,  d3,  d4,  d5},  [r2]!
        vadd.s16        d0,  d0,  d30
        vaddl.u8        q2,  d4,  d5
        vaddl.u8        q1,  d2,  d3
        vadd.u16        d4,  d4,  d5
        vadd.u16        d2,  d2,  d3
        vld1.8          {d16, d17, d18, d19}, [r2]
        vpadd.u16       d4,  d4
        vpadd.u16       d2,  d2
        vpadd.u16       d4,  d4
        vpadd.u16       d2,  d2
        vaddl.u8        q8,  d16, d17
        vaddl.u8        q9,  d18, d19
        vadd.u16        d16, d16, d17
        vadd.u16        d18, d18, d19
        vpadd.u16       d16, d16
        vpadd.u16       d18, d18
        vpadd.u16       d16, d16
        vpadd.u16       d18, d18
        vadd.u16        d2,  d2,  d4
        vadd.u16        d3,  d16, d18
        cmp             r4,  #64
        vadd.s16        d0,  d0,  d2
        vadd.s16        d0,  d0,  d3
        vshl.u16        d18, d0,  d28
        beq             1f
        // h = 16/32
        movw            lr,  #(0x5556/2)
        movt            lr,  #(0x3334/2)
        and             r5,  r4,  #31
        lsr             lr,  lr,  r5
        vdup.16         d30, lr
        vqdmulh.s16     d18, d18, d30
1:
        sub             r1,  r1,  #32
        vdup.8          q0,  d18[0]
        vdup.8          q1,  d18[0]
2:
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        subs            r4,  r4,  #4
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128]!
        vst1.8          {d0,  d1,  d2,  d3},  [r0,  :128], r1
        vst1.8          {d0,  d1,  d2,  d3},  [r12, :128], r1
        bgt             2b
        pop             {r4-r6, pc}
endfunc

// void ipred_paeth_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                            const pixel *const topleft,
//                            const int width, const int height, const int a,
//                            const int max_width, const int max_height);
function ipred_paeth_8bpc_neon, export=1
        push            {r4-r8, lr}
        ldr             r4,  [sp, #24]
        clz             lr,  r3
        adr             r5,  L(ipred_paeth_tbl)
        sub             lr,  lr,  #25
        ldr             lr,  [r5, lr, lsl #2]
        vld1.8          {d4[], d5[]},  [r2]
        add             r8,  r2,  #1
        sub             r2,  r2,  #4
        add             r5,  r5,  lr
        mov             r7,  #-4
        add             r6,  r0,  r1
        lsl             r1,  r1,  #1
        bx              r5

        .align 2
L(ipred_paeth_tbl):
        .word 640f - L(ipred_paeth_tbl) + CONFIG_THUMB
        .word 320f - L(ipred_paeth_tbl) + CONFIG_THUMB
        .word 160f - L(ipred_paeth_tbl) + CONFIG_THUMB
        .word 80f  - L(ipred_paeth_tbl) + CONFIG_THUMB
        .word 40f  - L(ipred_paeth_tbl) + CONFIG_THUMB

40:
        vld1.32         {d6[], d7[]},  [r8]
        vsubl.u8        q8,  d6,  d4  // top - topleft
4:
        vld4.8          {d0[], d1[], d2[], d3[]},  [r2, :32], r7
        vzip.32         d0,  d1
        vzip.32         d2,  d3
        vaddw.u8        q9,  q8,  d0
        vaddw.u8        q10, q8,  d2
        vqmovun.s16     d18, q9       // base
        vqmovun.s16     d19, q10
        vmov            d1,  d2
        vabd.u8         q10, q3,  q9  // tdiff
        vabd.u8         q11, q2,  q9  // tldiff
        vabd.u8         q9,  q0,  q9  // ldiff
        vmin.u8         q12, q10, q11 // min(tdiff, tldiff)
        vcge.u8         q10, q11, q10 // tldiff >= tdiff
        vcge.u8         q9,  q12, q9  // min(tdiff, tldiff) >= ldiff
        vbsl            q10, q3,  q2  // tdiff <= tldiff ? top : topleft
        vbit            q10, q0,  q9  // ldiff <= min ? left : ...
        vst1.32         {d21[1]}, [r0, :32], r1
        vst1.32         {d21[0]}, [r6, :32], r1
        subs            r4,  r4,  #4
        vst1.32         {d20[1]}, [r0, :32], r1
        vst1.32         {d20[0]}, [r6, :32], r1
        bgt             4b
        pop             {r4-r8, pc}
80:
        vld1.8          {d6},  [r8]
        vsubl.u8        q8,  d6,  d4  // top - topleft
        vmov            d7,  d6
8:
        vld4.8          {d0[], d1[], d2[], d3[]},  [r2, :32], r7
        vaddw.u8        q9,  q8,  d0
        vaddw.u8        q10, q8,  d1
        vaddw.u8        q11, q8,  d2
        vaddw.u8        q12, q8,  d3
        vqmovun.s16     d18, q9       // base
        vqmovun.s16     d19, q10
        vqmovun.s16     d20, q11
        vqmovun.s16     d21, q12
        vabd.u8         q11, q3,  q9  // tdiff
        vabd.u8         q12, q3,  q10
        vabd.u8         q13, q2,  q9  // tldiff
        vabd.u8         q14, q2,  q10
        vabd.u8         q10, q1,  q10 // ldiff
        vabd.u8         q9,  q0,  q9
        vmin.u8         q15, q12, q14 // min(tdiff, tldiff)
        vcge.u8         q12, q14, q12 // tldiff >= tdiff
        vmin.u8         q14, q11, q13 // min(tdiff, tldiff)
        vcge.u8         q11, q13, q11 // tldiff >= tdiff
        vcge.u8         q10, q15, q10 // min(tdiff, tldiff) >= ldiff
        vcge.u8         q9,  q14, q9
        vbsl            q12, q3,  q2  // tdiff <= tldiff ? top : topleft
        vbsl            q11, q3,  q2
        vbit            q12, q1,  q10 // ldiff <= min ? left : ...
        vbit            q11, q0,  q9
        vst1.8          {d25}, [r0, :64], r1
        vst1.8          {d24}, [r6, :64], r1
        subs            r4,  r4,  #4
        vst1.8          {d23}, [r0, :64], r1
        vst1.8          {d22}, [r6, :64], r1
        bgt             8b
        pop             {r4-r8, pc}
160:
320:
640:
        vld1.8          {d6},  [r8]!
        mov             r12, r3
        // Set up pointers for four rows in parallel; r0, r6, r5, lr
        add             r5,  r0,  r1
        add             lr,  r6,  r1
        lsl             r1,  r1,  #1
        sub             r1,  r1,  r3
1:
        vld4.8          {d0[], d1[], d2[], d3[]},  [r2, :32], r7
2:
        vsubl.u8        q8,  d6,  d4  // top - topleft
        vmov            d7,  d6
        vaddw.u8        q9,  q8,  d0
        vaddw.u8        q10, q8,  d1
        vaddw.u8        q11, q8,  d2
        vaddw.u8        q12, q8,  d3
        vqmovun.s16     d18, q9       // base
        vqmovun.s16     d19, q10
        vqmovun.s16     d20, q11
        vqmovun.s16     d21, q12
        vabd.u8         q11, q3,  q9  // tdiff
        vabd.u8         q12, q3,  q10
        vabd.u8         q13, q2,  q9  // tldiff
        vabd.u8         q14, q2,  q10
        vabd.u8         q10, q1,  q10 // ldiff
        vabd.u8         q9,  q0,  q9
        vmin.u8         q15, q12, q14 // min(tdiff, tldiff)
        vcge.u8         q12, q14, q12 // tldiff >= tdiff
        vmin.u8         q14, q11, q13 // min(tdiff, tldiff)
        vcge.u8         q11, q13, q11 // tldiff >= tdiff
        vcge.u8         q10, q15, q10 // min(tdiff, tldiff) >= ldiff
        vcge.u8         q9,  q14, q9
        vbsl            q12, q3,  q2  // tdiff <= tldiff ? top : topleft
        vbsl            q11, q3,  q2
        vbit            q12, q1,  q10 // ldiff <= min ? left : ...
        vbit            q11, q0,  q9
        subs            r3,  r3,  #8
        vst1.8          {d25}, [r0, :64]!
        vst1.8          {d24}, [r6, :64]!
        vst1.8          {d23}, [r5, :64]!
        vst1.8          {d22}, [lr, :64]!
        ble             8f
        vld1.8          {d6},  [r8]!
        b               2b
8:
        subs            r4,  r4,  #4
        ble             9f
        // End of horizontal loop, move pointers to next four rows
        sub             r8,  r8,  r12
        add             r0,  r0,  r1
        add             r6,  r6,  r1
        vld1.8          {d6},  [r8]!
        add             r5,  r5,  r1
        add             lr,  lr,  r1
        mov             r3,  r12
        b               1b
9:
        pop             {r4-r8, pc}
endfunc

// void ipred_smooth_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                             const pixel *const topleft,
//                             const int width, const int height, const int a,
//                             const int max_width, const int max_height);
function ipred_smooth_8bpc_neon, export=1
        push            {r4-r10, lr}
        ldr             r4,  [sp, #32]
        movrel          r10, X(sm_weights)
        add             r12, r10, r4
        add             r10, r10, r3
        clz             r9,  r3
        adr             r5,  L(ipred_smooth_tbl)
        sub             lr,  r2,  r4
        sub             r9,  r9,  #25
        ldr             r9,  [r5, r9, lsl #2]
        vld1.8          {d4[]},  [lr] // bottom
        add             r8,  r2,  #1
        add             r5,  r5,  r9
        add             r6,  r0,  r1
        lsl             r1,  r1,  #1
        bx              r5

        .align 2
L(ipred_smooth_tbl):
        .word 640f - L(ipred_smooth_tbl) + CONFIG_THUMB
        .word 320f - L(ipred_smooth_tbl) + CONFIG_THUMB
        .word 160f - L(ipred_smooth_tbl) + CONFIG_THUMB
        .word 80f  - L(ipred_smooth_tbl) + CONFIG_THUMB
        .word 40f  - L(ipred_smooth_tbl) + CONFIG_THUMB

40:
        vld1.32         {d16[]}, [r8]       // top
        vld1.32         {d18[]}, [r10, :32] // weights_hor
        sub             r2,  r2,  #4
        mov             r7,  #-4
        vdup.8          q3,  d16[3]   // right
        vsubl.u8        q8,  d16, d4  // top-bottom
        vmovl.u8        q9,  d18      // weights_hor
4:
        vld4.8          {d0[],  d1[],  d2[],  d3[]},  [r2,  :32], r7 // left
        vld4.8          {d20[], d21[], d22[], d23[]}, [r12, :32]!    // weights_ver
        vshll.i8        q12, d6,  #8  // right*256
        vshll.i8        q13, d6,  #8
        vzip.32         d1,  d0       // left, flipped
        vzip.32         d3,  d2
        vzip.32         d20, d21      // weights_ver
        vzip.32         d22, d23
        vshll.i8        q14, d4,  #8  // bottom*256
        vshll.i8        q15, d4,  #8
        vsubl.u8        q0,  d1,  d6  // left-right
        vsubl.u8        q1,  d3,  d6
        vmovl.u8        q10, d20      // weights_ver
        vmovl.u8        q11, d22
        vmla.i16        q12, q1,  q9  // right*256  + (left-right)*weights_hor
        vmla.i16        q13, q0,  q9  // (left flipped)
        vmla.i16        q14, q8,  q10 // bottom*256 + (top-bottom)*weights_ver
        vmla.i16        q15, q8,  q11
        vhadd.u16       q12, q12, q14
        vhadd.u16       q13, q13, q15
        vrshrn.i16      d24, q12, #8
        vrshrn.i16      d25, q13, #8
        vst1.32         {d24[0]}, [r0, :32], r1
        vst1.32         {d24[1]}, [r6, :32], r1
        subs            r4,  r4,  #4
        vst1.32         {d25[0]}, [r0, :32], r1
        vst1.32         {d25[1]}, [r6, :32], r1
        bgt             4b
        pop             {r4-r10, pc}
80:
        vld1.8          {d16}, [r8]       // top
        vld1.8          {d18}, [r10, :64] // weights_hor
        sub             r2,  r2,  #2
        mov             r7,  #-2
        vdup.8          q3,  d16[7]   // right
        vsubl.u8        q8,  d16, d4  // top-bottom
        vmovl.u8        q9,  d18      // weights_hor
8:
        vld2.8          {d0[],  d1[]},  [r2,  :16], r7 // left
        vld2.8          {d20[], d22[]}, [r12, :16]!    // weights_ver
        vshll.i8        q12, d6,  #8  // right*256
        vshll.i8        q13, d6,  #8
        vshll.i8        q14, d4,  #8  // bottom*256
        vshll.i8        q15, d4,  #8
        vsubl.u8        q1,  d0,  d6  // left-right (left flipped)
        vsubl.u8        q0,  d1,  d6
        vmovl.u8        q10, d20      // weights_ver
        vmovl.u8        q11, d22
        vmla.i16        q12, q0,  q9  // right*256  + (left-right)*weights_hor
        vmla.i16        q13, q1,  q9
        vmla.i16        q14, q8,  q10 // bottom*256 + (top-bottom)*weights_ver
        vmla.i16        q15, q8,  q11
        vhadd.u16       q12, q12, q14
        vhadd.u16       q13, q13, q15
        vrshrn.i16      d24, q12, #8
        vrshrn.i16      d25, q13, #8
        subs            r4,  r4,  #2
        vst1.8          {d24}, [r0, :64], r1
        vst1.8          {d25}, [r6, :64], r1
        bgt             8b
        pop             {r4-r10, pc}
160:
320:
640:
        add             lr,  r2,  r3
        sub             r2,  r2,  #2
        mov             r7,  #-2
        vld1.8          {d6[], d7[]}, [lr] // right
        sub             r1,  r1,  r3
        mov             r9,  r3

1:
        vld2.8          {d0[],  d1[]},  [r2,  :16], r7 // left
        vld2.8          {d20[], d22[]}, [r12, :16]!    // weights_ver
        vsubl.u8        q1,  d0,  d6  // left-right (left flipped)
        vsubl.u8        q0,  d1,  d6
        vmovl.u8        q10, d20      // weights_ver
        vmovl.u8        q11, d22
2:
        vld1.8          {d16}, [r8]!       // top
        vld1.8          {d18}, [r10, :64]! // weights_hor
        vshll.i8        q12, d6,  #8  // right*256
        vshll.i8        q13, d6,  #8
        vmovl.u8        q9,  d18      // weights_hor
        vshll.i8        q14, d4,  #8  // bottom*256
        vshll.i8        q15, d4,  #8
        vsubl.u8        q8,  d16, d4  // top-bottom
        vmla.i16        q12, q0,  q9  // right*256  + (left-right)*weights_hor
        vmla.i16        q13, q1,  q9
        vmla.i16        q14, q8,  q10 // bottom*256 + (top-bottom)*weights_ver
        vmla.i16        q15, q8,  q11
        vhadd.u16       q12, q12, q14
        vhadd.u16       q13, q13, q15
        vrshrn.i16      d24, q12, #8
        vrshrn.i16      d25, q13, #8
        subs            r3,  r3,  #8
        vst1.8          {d24}, [r0, :64]!
        vst1.8          {d25}, [r6, :64]!
        bgt             2b
        subs            r4,  r4,  #2
        ble             9f
        sub             r8,  r8,  r9
        sub             r10, r10, r9
        add             r0,  r0,  r1
        add             r6,  r6,  r1
        mov             r3,  r9
        b               1b
9:
        pop             {r4-r10, pc}
endfunc

// void ipred_smooth_v_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                               const pixel *const topleft,
//                               const int width, const int height, const int a,
//                               const int max_width, const int max_height);
function ipred_smooth_v_8bpc_neon, export=1
        push            {r4-r7, lr}
        ldr             r4,  [sp, #20]
        movrel          r7,  X(sm_weights)
        add             r7,  r7,  r4
        clz             lr,  r3
        adr             r5,  L(ipred_smooth_v_tbl)
        sub             r12, r2,  r4
        sub             lr,  lr,  #25
        ldr             lr,  [r5, lr, lsl #2]
        vld1.8          {d4[]},  [r12] // bottom
        add             r2,  r2,  #1
        add             r5,  r5,  lr
        add             r6,  r0,  r1
        lsl             r1,  r1,  #1
        bx              r5

        .align 2
L(ipred_smooth_v_tbl):
        .word 640f - L(ipred_smooth_v_tbl) + CONFIG_THUMB
        .word 320f - L(ipred_smooth_v_tbl) + CONFIG_THUMB
        .word 160f - L(ipred_smooth_v_tbl) + CONFIG_THUMB
        .word 80f  - L(ipred_smooth_v_tbl) + CONFIG_THUMB
        .word 40f  - L(ipred_smooth_v_tbl) + CONFIG_THUMB

40:
        vld1.32         {d6[]}, [r2]  // top
        vsubl.u8        q3,  d6,  d4  // top-bottom
4:
        vld4.8          {d16[], d17[], d18[], d19[]}, [r7, :32]! // weights_ver
        vshll.i8        q10, d4,  #8  // bottom*256
        vshll.i8        q11, d4,  #8
        vzip.32         d16, d17      // weights_ver
        vzip.32         d18, d19
        vmovl.u8        q8,  d16      // weights_ver
        vmovl.u8        q9,  d18
        subs            r4,  r4,  #4
        vmla.i16        q10, q3,  q8  // bottom*256 + (top-bottom)*weights_ver
        vmla.i16        q11, q3,  q9
        vrshrn.i16      d20, q10, #8
        vrshrn.i16      d21, q11, #8
        vst1.32         {d20[0]}, [r0, :32], r1
        vst1.32         {d20[1]}, [r6, :32], r1
        vst1.32         {d21[0]}, [r0, :32], r1
        vst1.32         {d21[1]}, [r6, :32], r1
        bgt             4b
        pop             {r4-r7, pc}
80:
        vld1.8          {d6}, [r2]    // top
        vsubl.u8        q3,  d6,  d4  // top-bottom
8:
        vld4.8          {d16[], d18[], d20[], d22[]}, [r7, :32]! // weights_ver
        vshll.i8        q12, d4,  #8  // bottom*256
        vshll.i8        q13, d4,  #8
        vshll.i8        q14, d4,  #8
        vshll.i8        q15, d4,  #8
        vmovl.u8        q8,  d16      // weights_ver
        vmovl.u8        q9,  d18
        vmovl.u8        q10, d20
        vmovl.u8        q11, d22
        vmla.i16        q12, q3,  q8  // bottom*256 + (top-bottom)*weights_ver
        vmla.i16        q13, q3,  q9
        vmla.i16        q14, q3,  q10
        vmla.i16        q15, q3,  q11
        vrshrn.i16      d24, q12, #8
        vrshrn.i16      d25, q13, #8
        vrshrn.i16      d26, q14, #8
        vrshrn.i16      d27, q15, #8
        vst1.8          {d24}, [r0, :64], r1
        vst1.8          {d25}, [r6, :64], r1
        subs            r4,  r4,  #4
        vst1.8          {d26}, [r0, :64], r1
        vst1.8          {d27}, [r6, :64], r1
        bgt             8b
        pop             {r4-r7, pc}
160:
320:
640:
        vpush           {q4-q7}
        // Set up pointers for four rows in parallel; r0, r6, r5, lr
        add             r5,  r0,  r1
        add             lr,  r6,  r1
        lsl             r1,  r1,  #1
        sub             r1,  r1,  r3
        mov             r12, r3

1:
        vld4.8          {d8[], d10[], d12[], d14[]}, [r7, :32]! // weights_ver
        vmovl.u8        q4,  d8       // weights_ver
        vmovl.u8        q5,  d10
        vmovl.u8        q6,  d12
        vmovl.u8        q7,  d14
2:
        vld1.8          {q3}, [r2]!   // top
        vshll.i8        q8,  d4,  #8  // bottom*256
        vshll.i8        q9,  d4,  #8
        vshll.i8        q10, d4,  #8
        vshll.i8        q11, d4,  #8
        vsubl.u8        q0,  d6,  d4  // top-bottom
        vsubl.u8        q1,  d7,  d4
        vshll.i8        q12, d4,  #8
        vshll.i8        q13, d4,  #8
        vshll.i8        q14, d4,  #8
        vshll.i8        q15, d4,  #8
        vmla.i16        q8,  q0,  q4  // bottom*256 + (top-bottom)*weights_ver
        vmla.i16        q9,  q1,  q4
        vmla.i16        q10, q0,  q5
        vmla.i16        q11, q1,  q5
        vmla.i16        q12, q0,  q6  // bottom*256 + (top-bottom)*weights_ver
        vmla.i16        q13, q1,  q6
        vmla.i16        q14, q0,  q7
        vmla.i16        q15, q1,  q7
        vrshrn.i16      d16, q8,  #8
        vrshrn.i16      d17, q9,  #8
        vrshrn.i16      d18, q10, #8
        vrshrn.i16      d19, q11, #8
        vrshrn.i16      d20, q12, #8
        vrshrn.i16      d21, q13, #8
        vrshrn.i16      d22, q14, #8
        vrshrn.i16      d23, q15, #8
        subs            r3,  r3,  #16
        vst1.8          {q8},  [r0, :128]!
        vst1.8          {q9},  [r6, :128]!
        vst1.8          {q10}, [r5, :128]!
        vst1.8          {q11}, [lr, :128]!
        bgt             2b
        subs            r4,  r4,  #4
        ble             9f
        sub             r2,  r2,  r12
        add             r0,  r0,  r1
        add             r6,  r6,  r1
        add             r5,  r5,  r1
        add             lr,  lr,  r1
        mov             r3,  r12
        b               1b
9:
        vpop            {q4-q7}
        pop             {r4-r7, pc}
endfunc

// void ipred_smooth_h_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                               const pixel *const topleft,
//                               const int width, const int height, const int a,
//                               const int max_width, const int max_height);
function ipred_smooth_h_8bpc_neon, export=1
        push            {r4-r8, lr}
        ldr             r4,  [sp, #24]
        movrel          r8,  X(sm_weights)
        add             r8,  r8,  r3
        clz             lr,  r3
        adr             r5,  L(ipred_smooth_h_tbl)
        add             r12, r2,  r3
        sub             lr,  lr,  #25
        ldr             lr,  [r5, lr, lsl #2]
        vld1.8          {d4[]},  [r12] // right
        add             r5,  r5,  lr
        add             r6,  r0,  r1
        lsl             r1,  r1,  #1
        bx              r5

        .align 2
L(ipred_smooth_h_tbl):
        .word 640f - L(ipred_smooth_h_tbl) + CONFIG_THUMB
        .word 320f - L(ipred_smooth_h_tbl) + CONFIG_THUMB
        .word 160f - L(ipred_smooth_h_tbl) + CONFIG_THUMB
        .word 80f  - L(ipred_smooth_h_tbl) + CONFIG_THUMB
        .word 40f  - L(ipred_smooth_h_tbl) + CONFIG_THUMB

40:
        vld1.32         {d6[]}, [r8, :32] // weights_hor
        sub             r2,  r2,  #4
        mov             r7,  #-4
        vmovl.u8        q3,  d6       // weights_hor
4:
        vld4.8          {d0[], d1[], d2[], d3[]},  [r2, :32], r7 // left
        vshll.i8        q8,  d4,  #8  // right*256
        vshll.i8        q9,  d4,  #8
        vzip.32         d3,  d2       // left, flipped
        vzip.32         d1,  d0
        vsubl.u8        q1,  d3,  d4  // left-right
        vsubl.u8        q0,  d1,  d4
        subs            r4,  r4,  #4
        vmla.i16        q8,  q1,  q3  // right*256  + (left-right)*weights_hor
        vmla.i16        q9,  q0,  q3
        vrshrn.i16      d16, q8,  #8
        vrshrn.i16      d17, q9,  #8
        vst1.32         {d16[0]}, [r0, :32], r1
        vst1.32         {d16[1]}, [r6, :32], r1
        vst1.32         {d17[0]}, [r0, :32], r1
        vst1.32         {d17[1]}, [r6, :32], r1
        bgt             4b
        pop             {r4-r8, pc}
80:
        vld1.8          {d6}, [r8, :64] // weights_hor
        sub             r2,  r2,  #4
        mov             r7,  #-4
        vmovl.u8        q3,  d6       // weights_hor
8:
        vld4.8          {d16[], d18[], d20[], d22[]},  [r2, :32], r7 // left
        vshll.i8        q12, d4,  #8  // right*256
        vshll.i8        q13, d4,  #8
        vshll.i8        q14, d4,  #8
        vshll.i8        q15, d4,  #8
        vsubl.u8        q11, d22, d4  // left-right
        vsubl.u8        q10, d20, d4
        vsubl.u8        q9,  d18, d4
        vsubl.u8        q8,  d16, d4
        vmla.i16        q12, q11, q3  // right*256  + (left-right)*weights_hor
        vmla.i16        q13, q10, q3  // (left flipped)
        vmla.i16        q14, q9,  q3
        vmla.i16        q15, q8,  q3
        vrshrn.i16      d24, q12, #8
        vrshrn.i16      d25, q13, #8
        vrshrn.i16      d26, q14, #8
        vrshrn.i16      d27, q15, #8
        vst1.8          {d24}, [r0, :64], r1
        vst1.8          {d25}, [r6, :64], r1
        subs            r4,  r4,  #4
        vst1.8          {d26}, [r0, :64], r1
        vst1.8          {d27}, [r6, :64], r1
        bgt             8b
        pop             {r4-r8, pc}
160:
320:
640:
        vpush           {q4-q7}
        sub             r2,  r2,  #4
        mov             r7,  #-4
        // Set up pointers for four rows in parallel; r0, r6, r5, lr
        add             r5,  r0,  r1
        add             lr,  r6,  r1
        lsl             r1,  r1,  #1
        sub             r1,  r1,  r3
        mov             r12, r3

1:
        vld4.8          {d8[], d10[], d12[], d14[]},  [r2, :32], r7 // left
        vsubl.u8        q4,  d8,  d4  // left-right
        vsubl.u8        q5,  d10, d4
        vsubl.u8        q6,  d12, d4
        vsubl.u8        q7,  d14, d4
2:
        vld1.8          {q1}, [r8, :128]! // weights_hor
        vshll.i8        q8,  d4,  #8  // right*256
        vshll.i8        q9,  d4,  #8
        vshll.i8        q10, d4,  #8
        vshll.i8        q11, d4,  #8
        vmovl.u8        q0,  d2       // weights_hor
        vmovl.u8        q1,  d3
        vshll.i8        q12, d4,  #8
        vshll.i8        q13, d4,  #8
        vshll.i8        q14, d4,  #8
        vshll.i8        q15, d4,  #8
        vmla.i16        q8,  q7,  q0  // right*256  + (left-right)*weights_hor
        vmla.i16        q9,  q7,  q1  // (left flipped)
        vmla.i16        q10, q6,  q0
        vmla.i16        q11, q6,  q1
        vmla.i16        q12, q5,  q0
        vmla.i16        q13, q5,  q1
        vmla.i16        q14, q4,  q0
        vmla.i16        q15, q4,  q1
        vrshrn.i16      d16, q8,  #8
        vrshrn.i16      d17, q9,  #8
        vrshrn.i16      d18, q10, #8
        vrshrn.i16      d19, q11, #8
        vrshrn.i16      d20, q12, #8
        vrshrn.i16      d21, q13, #8
        vrshrn.i16      d22, q14, #8
        vrshrn.i16      d23, q15, #8
        subs            r3,  r3,  #16
        vst1.8          {q8},  [r0, :128]!
        vst1.8          {q9},  [r6, :128]!
        vst1.8          {q10}, [r5, :128]!
        vst1.8          {q11}, [lr, :128]!
        bgt             2b
        subs            r4,  r4,  #4
        ble             9f
        sub             r8,  r8,  r12
        add             r0,  r0,  r1
        add             r6,  r6,  r1
        add             r5,  r5,  r1
        add             lr,  lr,  r1
        mov             r3,  r12
        b               1b
9:
        vpop            {q4-q7}
        pop             {r4-r8, pc}
endfunc

// void ipred_filter_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                             const pixel *const topleft,
//                             const int width, const int height, const int filt_idx,
//                             const int max_width, const int max_height);
function ipred_filter_8bpc_neon, export=1
        push            {r4-r8, lr}
        movw            r12, #511
        ldrd            r4,  r5,  [sp, #24]
        and             r5,  r5,  r12 // 511
        movrel          r6,  X(filter_intra_taps)
        lsl             r5,  r5,  #6
        add             r6,  r6,  r5
        vld1.8          {d20, d21, d22, d23}, [r6, :128]!
        clz             lr,  r3
        adr             r5,  L(ipred_filter_tbl)
        vld1.8          {d27, d28, d29}, [r6, :64]
        sub             lr,  lr,  #26
        ldr             lr,  [r5, lr, lsl #2]
        vmovl.s8        q8,  d20
        vmovl.s8        q9,  d21
        add             r5,  r5,  lr
        vmovl.s8        q10, d22
        vmovl.s8        q11, d23
        add             r6,  r0,  r1
        lsl             r1,  r1,  #1
        vmovl.s8        q12, d27
        vmovl.s8        q13, d28
        vmovl.s8        q14, d29
        add             r8,  r2,  #1
        sub             r2,  r2,  #2
        mov             r7,  #-2
        bx              r5

        .align 2
L(ipred_filter_tbl):
        .word 320f - L(ipred_filter_tbl) + CONFIG_THUMB
        .word 160f - L(ipred_filter_tbl) + CONFIG_THUMB
        .word 80f  - L(ipred_filter_tbl) + CONFIG_THUMB
        .word 40f  - L(ipred_filter_tbl) + CONFIG_THUMB

40:
        vld1.32         {d0[]}, [r8]     // top (0-3)
        vmovl.u8        q0,  d0          // top (0-3)
4:
        vld1.32         {d2[]}, [r2], r7 // left (0-1) + topleft (2)
        vmul.i16        q2,  q9,  d0[0]  // p1(top[0]) * filter(1)
        vmla.i16        q2,  q10, d0[1]  // p2(top[1]) * filter(2)
        vmla.i16        q2,  q11, d0[2]  // p3(top[2]) * filter(3)
        vmovl.u8        q1,  d2          // left (0-1) + topleft (2)
        vmla.i16        q2,  q12, d0[3]  // p4(top[3]) * filter(4)
        vmla.i16        q2,  q8,  d2[2]  // p0(topleft) * filter(0)
        vmla.i16        q2,  q13, d2[1]  // p5(left[0]) * filter(5)
        vmla.i16        q2,  q14, d2[0]  // p6(left[1]) * filter(6)
        vqrshrun.s16    d4,  q2,  #4
        subs            r4,  r4,  #2
        vst1.32         {d4[0]}, [r0, :32], r1
        vmovl.u8        q0,  d4
        vst1.32         {d4[1]}, [r6, :32], r1
        vmov            d0,  d1          // move top from [4-7] to [0-3]
        bgt             4b
        pop             {r4-r8, pc}
80:
        vld1.8          {d0},  [r8]      // top (0-7)
        vmovl.u8        q0,  d0          // top (0-7)
8:
        vld1.32         {d2[]}, [r2], r7 // left (0-1) + topleft (2)
        vmul.i16        q2,  q9,  d0[0]  // p1(top[0]) * filter(1)
        vmla.i16        q2,  q10, d0[1]  // p2(top[1]) * filter(2)
        vmla.i16        q2,  q11, d0[2]  // p3(top[2]) * filter(3)
        vmovl.u8        q1,  d2          // left (0-1) + topleft (2)
        vmla.i16        q2,  q12, d0[3]  // p4(top[3]) * filter(4)
        vmla.i16        q2,  q8,  d2[2]  // p0(topleft) * filter(0)
        vmla.i16        q2,  q13, d2[1]  // p5(left[0]) * filter(5)
        vmla.i16        q2,  q14, d2[0]  // p6(left[1]) * filter(6)
        vmul.i16        q3,  q9,  d1[0]  // p1(top[0]) * filter(1)
        vmla.i16        q3,  q10, d1[1]  // p2(top[1]) * filter(2)
        vmla.i16        q3,  q11, d1[2]  // p3(top[2]) * filter(3)
        vqrshrun.s16    d4,  q2,  #4
        vmovl.u8        q1,  d4          // first block, in 16 bit
        vmla.i16        q3,  q12, d1[3]  // p4(top[3]) * filter(4)
        vmla.i16        q3,  q8,  d0[3]  // p0(topleft) * filter(0)
        vmla.i16        q3,  q13, d2[3]  // p5(left[0]) * filter(5)
        vmla.i16        q3,  q14, d3[3]  // p6(left[1]) * filter(6)
        vqrshrun.s16    d5,  q3,  #4
        vzip.32         d4,  d5
        subs            r4,  r4,  #2
        vst1.8          {d4}, [r0, :64], r1
        vmovl.u8        q0,  d5
        vst1.8          {d5}, [r6, :64], r1
        bgt             8b
        pop             {r4-r8, pc}
160:
320:
        vpush           {q4-q5}
        sub             r1,  r1,  r3
        mov             lr,  r3

1:
        vld1.32         {d0[]}, [r2], r7 // left (0-1) + topleft (2)
        vmovl.u8        q0,  d0          // left (0-1) + topleft (2)
2:
        vld1.8          {q2}, [r8]!      // top(0-15)
        vmul.i16        q3,  q8,  d0[2]  // p0(topleft) * filter(0)
        vmla.i16        q3,  q13, d0[1]  // p5(left[0]) * filter(5)
        vmovl.u8        q1,  d4          // top(0-7)
        vmovl.u8        q2,  d5          // top(8-15)
        vmla.i16        q3,  q14, d0[0]  // p6(left[1]) * filter(6)
        vmla.i16        q3,  q9,  d2[0]  // p1(top[0]) * filter(1)
        vmla.i16        q3,  q10, d2[1]  // p2(top[1]) * filter(2)
        vmla.i16        q3,  q11, d2[2]  // p3(top[2]) * filter(3)
        vmla.i16        q3,  q12, d2[3]  // p4(top[3]) * filter(4)

        vmul.i16        q4,  q9,  d3[0]  // p1(top[0]) * filter(1)
        vmla.i16        q4,  q10, d3[1]  // p2(top[1]) * filter(2)
        vmla.i16        q4,  q11, d3[2]  // p3(top[2]) * filter(3)
        vqrshrun.s16    d6,  q3,  #4
        vmovl.u8        q0,  d6          // first block, in 16 bit
        vmla.i16        q4,  q12, d3[3]  // p4(top[3]) * filter(4)
        vmla.i16        q4,  q8,  d2[3]  // p0(topleft) * filter(0)
        vmla.i16        q4,  q13, d0[3]  // p5(left[0]) * filter(5)
        vmla.i16        q4,  q14, d1[3]  // p6(left[1]) * filter(6)

        vmul.i16        q5,  q9,  d4[0]  // p1(top[0]) * filter(1)
        vmla.i16        q5,  q10, d4[1]  // p2(top[1]) * filter(2)
        vmla.i16        q5,  q11, d4[2]  // p3(top[2]) * filter(3)
        vqrshrun.s16    d7,  q4,  #4
        vmovl.u8        q0,  d7          // second block, in 16 bit
        vmla.i16        q5,  q12, d4[3]  // p4(top[3]) * filter(4)
        vmla.i16        q5,  q8,  d3[3]  // p0(topleft) * filter(0)
        vmla.i16        q5,  q13, d0[3]  // p5(left[0]) * filter(5)
        vmla.i16        q5,  q14, d1[3]  // p6(left[1]) * filter(6)

        vmul.i16        q15, q9,  d5[0]  // p1(top[0]) * filter(1)
        vmla.i16        q15, q10, d5[1]  // p2(top[1]) * filter(2)
        vmla.i16        q15, q11, d5[2]  // p3(top[2]) * filter(3)
        vqrshrun.s16    d8,  q5,  #4
        vmovl.u8        q0,  d8          // third block, in 16 bit
        vmov.u8         r12, d5[6]
        vmla.i16        q15, q12, d5[3]  // p4(top[3]) * filter(4)
        vmla.i16        q15, q8,  d4[3]  // p0(topleft) * filter(0)
        vmla.i16        q15, q13, d0[3]  // p5(left[0]) * filter(5)
        vmla.i16        q15, q14, d1[3]  // p6(left[1]) * filter(6)
        vmov.8          d0[4], r12

        subs            r3,  r3,  #16
        vqrshrun.s16    d9,  q15, #4

        vst4.32         {d6[0], d7[0], d8[0], d9[0]}, [r0, :128]!
        vst4.32         {d6[1], d7[1], d8[1], d9[1]}, [r6, :128]!
        ble             8f
        vmov.u8         r12, d9[7]
        vmov.8          d0[0], r12
        vmov.u8         r12, d9[3]
        vmov.8          d0[2], r12
        b               2b
8:
        subs            r4,  r4,  #2

        ble             9f
        sub             r8,  r6,  lr
        add             r0,  r0,  r1
        add             r6,  r6,  r1
        mov             r3,  lr
        b               1b
9:
        vpop            {q4-q5}
        pop             {r4-r8, pc}
endfunc

// void pal_pred_8bpc_neon(pixel *dst, const ptrdiff_t stride,
//                         const pixel *const pal, const uint8_t *idx,
//                         const int w, const int h);
function pal_pred_8bpc_neon, export=1
        push            {r4-r5, lr}
        ldrd            r4,  r5,  [sp, #12]
        vld1.8          {d0}, [r2, :64]
        clz             lr,  r4
        adr             r12, L(pal_pred_tbl)
        sub             lr,  lr,  #25
        vmov.i8         q15, #7
        ldr             lr,  [r12, lr, lsl #2]
        add             r12, r12, lr
        add             r2,  r0,  r1
        bx              r12

        .align 2
L(pal_pred_tbl):
        .word 640f - L(pal_pred_tbl) + CONFIG_THUMB
        .word 320f - L(pal_pred_tbl) + CONFIG_THUMB
        .word 160f - L(pal_pred_tbl) + CONFIG_THUMB
        .word 80f  - L(pal_pred_tbl) + CONFIG_THUMB
        .word 40f  - L(pal_pred_tbl) + CONFIG_THUMB

40:
        lsl             r1,  r1,  #1
4:
        vld1.8          {d2}, [r3, :64]!
        subs            r5,  r5,  #4
        vshr.u8         d3,  d2,  #4
        vand.u8         d2,  d2,  d30
        vzip.8          d2,  d3
        vtbl.8          d2, {d0}, d2
        vtbl.8          d3, {d0}, d3
        vst1.32         {d2[0]}, [r0, :32], r1
        vst1.32         {d2[1]}, [r2, :32], r1
        vst1.32         {d3[0]}, [r0, :32], r1
        vst1.32         {d3[1]}, [r2, :32], r1
        bgt             4b
        pop             {r4-r5, pc}
80:
        lsl             r1,  r1,  #1
8:
        vld1.8          {q1}, [r3, :64]!
        subs            r5,  r5,  #4
        vshr.u8         q2,  q1,  #4
        vand.u8         q1,  q1,  q15
        vzip.8          q1,  q2
        vtbl.8          d2, {d0}, d2
        vtbl.8          d3, {d0}, d3
        vst1.8          {d2}, [r0, :64], r1
        vtbl.8          d4, {d0}, d4
        vst1.8          {d3}, [r2, :64], r1
        vtbl.8          d5, {d0}, d5
        vst1.8          {d4}, [r0, :64], r1
        vst1.8          {d5}, [r2, :64], r1
        bgt             8b
        pop             {r4-r5, pc}
160:
        lsl             r1,  r1,  #1
16:
        vld1.8          {q10, q11}, [r3, :64]!
        subs            r5,  r5,  #4
        vand.u8         q8,  q10, q15
        vshr.u8         q9,  q10, #4
        vand.u8         q10, q11, q15
        vshr.u8         q11, q11, #4
        vzip.8          q8,  q9
        vzip.8          q10, q11
        vtbl.8          d16, {d0}, d16
--> --------------------

--> maximum size reached

--> --------------------