| Quellcodebibliothek Statistik Leitseite products/Sources/formale Sprachen/C/Firefox/third_party/dav1d/src/x86/ (Browser von der Mozilla Stiftung Version 136.0.1©) Datei vom 10.2.2025 mit Größe 227 kB |
|
Quelle itx16_avx512.asmSprache: Masm |
|||||||||||||||
|
; Copyright © 2022-2023, VideoLAN and dav1d authors ; Copyright © 2022-2023, Two Orioles, LLC ; All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions are met: ; ; 1. Redistributions of source code must retain the above copyright notice, this ; list of conditions and the following disclaimer. ; ; 2. Redistributions in binary form must reproduce the above copyright notice, ; this list of conditions and the following disclaimer in the documentation ; and/or other materials provided with the distribution. ; ; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED ; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE ; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES ; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; ; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS ; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %include "config.asm" %include "ext/x86/x86inc.asm" %if ARCH_X86_64 SECTION_RODATA 64 idct8x8p: db 0, 1, 4, 5, 2, 3, 6, 7, 16, 17, 20, 21, 18, 19, 22, 23 db 8, 9, 12, 13, 10, 11, 14, 15, 24, 25, 28, 29, 26, 27, 30, 31 db 32, 33, 36, 37, 34, 35, 38, 39, 48, 49, 52, 53, 50, 51, 54, 55 db 40, 41, 44, 45, 42, 43, 46, 47, 56, 57, 60, 61, 58, 59, 62, 63 idtx8x8p: db 0, 1, 32, 33, 2, 3, 34, 35, 4, 5, 36, 37, 6, 7, 38, 39 db 8, 9, 40, 41, 10, 11, 42, 43, 12, 13, 44, 45, 14, 15, 46, 47 db 16, 17, 48, 49, 18, 19, 50, 51, 20, 21, 52, 53, 22, 23, 54, 55 db 24, 25, 56, 57, 26, 27, 58, 59, 28, 29, 60, 61, 30, 31, 62, 63 idct8x16p: db 54, 55, 2, 3, 22, 23, 34, 35, 38, 39, 18, 19, 6, 7, 50, 51 db 62, 63, 10, 11, 30, 31, 42, 43, 46, 47, 26, 27, 14, 15, 58, 59 db 52, 53, 4, 5, 20, 21, 36, 37, 32, 33, 0, 1, 48, 49, 16, 17 db 60, 61, 12, 13, 28, 29, 44, 45, 40, 41, 8, 9, 56, 57, 24, 25 iadst8x16p: db 0, 1, 54, 55, 48, 49, 6, 7, 16, 17, 38, 39, 32, 33, 22, 23 db 8, 9, 62, 63, 56, 57, 14, 15, 24, 25, 46, 47, 40, 41, 30, 31 db 4, 5, 50, 51, 52, 53, 2, 3, 20, 21, 34, 35, 36, 37, 18, 19 db 12, 13, 58, 59, 60, 61, 10, 11, 28, 29, 42, 43, 44, 45, 26, 27 permA: db 0, 1, 0, 8, 4, 5, 1, 9, 8, 9, 4, 12, 12, 13, 5, 13 db 16, 17, 16, 24, 20, 21, 17, 25, 24, 25, 20, 28, 28, 29, 21, 29 db 2, 3, 2, 10, 6, 7, 3, 11, 10, 11, 6, 14, 14, 15, 7, 15 db 18, 19, 18, 26, 22, 23, 19, 27, 26, 27, 22, 30, 30, 31, 23, 31 permB: db 4, 2, 1, 8, 0, 0, 1, 0, 12, 3, 3, 10, 8, 1, 3, 2 db 5, 10, 5, 12, 1, 8, 5, 4, 13, 11, 7, 14, 9, 9, 7, 6 db 6, 6, 13, 4, 2, 4, 4, 5, 14, 7, 15, 6, 10, 5, 6, 7 db 7, 14, 9, 0, 3, 12, 0, 1, 15, 15, 11, 2, 11, 13, 2, 3 permC: db 0, 9, 0, 0, 0, 1, 4, 4, 2, 11, 2, 2, 2, 3, 6, 6 db 1, 8, 1, 8, 4, 5, 5, 12, 3, 10, 3, 10, 6, 7, 7, 14 db 9, 1, 8, 1, 1, 0, 12, 5, 11, 3, 10, 3, 3, 2, 14, 7 db 8, 0, 9, 9, 5, 4, 13, 13, 10, 2, 11, 11, 7, 6, 15, 15 idct8x32p: db 0, 1, 4, 5, 16, 17, 20, 21, 32, 33, 36, 37, 48, 49, 52, 53 db 8, 9, 12, 13, 24, 25, 28, 29, 40, 41, 44, 45, 56, 57, 60, 61 db 2, 3, 6, 7, 18, 19, 22, 23, 34, 35, 38, 39, 50, 51, 54, 55 db 10, 11, 14, 15, 26, 27, 30, 31, 42, 43, 46, 47, 58, 59, 62, 63 idct32x8p: db 2, 18, 0, 16, 3, 19, 1, 17, 10, 26, 8, 24, 11, 27, 9, 25 db 34, 50, 32, 48, 35, 51, 33, 49, 42, 58, 40, 56, 43, 59, 41, 57 db 6, 22, 4, 20, 7, 23, 5, 21, 14, 30, 12, 28, 15, 31, 13, 29 db 38, 54, 36, 52, 39, 55, 37, 53, 46, 62, 44, 60, 47, 63, 45, 61 idtx32x8p: db 0, 8, 16, 24, 4, 12, 20, 28, 2, 10, 18, 26, 6, 14, 22, 30 db 32, 40, 48, 56, 36, 44, 52, 60, 34, 42, 50, 58, 38, 46, 54, 62 db 1, 9, 17, 25, 5, 13, 21, 29, 3, 11, 19, 27, 7, 15, 23, 31 db 33, 41, 49, 57, 37, 45, 53, 61, 35, 43, 51, 59, 39, 47, 55, 63 pw_2048_m2048: times 16 dw 2048 pw_m2048_2048: times 16 dw -2048 pw_2048: times 16 dw 2048 ; flags: 0 = ++, 1 = +-, 2 = -+, 3 = ++-, 4=-- %macro COEF_PAIR 2-3 0 ; a, b, flags %if %3 == 1 pd_%1_m%2: dd %1, %1, -%2, -%2 %define pd_%1 (pd_%1_m%2 + 4*0) %define pd_m%2 (pd_%1_m%2 + 4*2) %elif %3 == 2 pd_m%1_%2: dd -%1, -%1, %2, %2 %define pd_m%1 (pd_m%1_%2 + 4*0) %define pd_%2 (pd_m%1_%2 + 4*2) %elif %3 == 4 pd_m%1_m%2: dd -%1, -%1, -%2, -%2 %define pd_m%1 (pd_m%1_m%2 + 4*0) %define pd_m%2 (pd_m%1_m%2 + 4*2) %else pd_%1_%2: dd %1, %1, %2, %2 %define pd_%1 (pd_%1_%2 + 4*0) %define pd_%2 (pd_%1_%2 + 4*2) %if %3 == 3 %define pd_%2_m%2 pd_%2 dd -%2, -%2 %endif %endif %endmacro COEF_PAIR 101, 501 COEF_PAIR 201, 601, 1 COEF_PAIR 201, 995 COEF_PAIR 401, 1189, 1 COEF_PAIR 401, 1931 COEF_PAIR 401, 3920 COEF_PAIR 401, 4076 COEF_PAIR 700, 301, 4 COEF_PAIR 799, 2276, 1 COEF_PAIR 799, 3406 COEF_PAIR 799, 4017 COEF_PAIR 1380, 601 COEF_PAIR 1751, 2440 COEF_PAIR 2598, 1189 COEF_PAIR 2598, 1931, 2 COEF_PAIR 2598, 3612 COEF_PAIR 2751, 2106 COEF_PAIR 2896, 1567, 3 COEF_PAIR 2896, 3784, 3 COEF_PAIR 3035, 3513 COEF_PAIR 3166, 1931 COEF_PAIR 3166, 3612 COEF_PAIR 3166, 3920 COEF_PAIR 3703, 3290 COEF_PAIR 3857, 4052 COEF_PAIR 4017, 2276 COEF_PAIR 4017, 3406 COEF_PAIR 4036, 4085 COEF_PAIR 4076, 1189 COEF_PAIR 4076, 3612 COEF_PAIR 4076, 3920 COEF_PAIR 4091, 3973 COEF_PAIR 4091, 4052 COEF_PAIR 4095, 4065 pb_32: times 4 db 32 pw_5: times 2 dw 5 pw_4096: times 2 dw 4096 pw_8192: times 2 dw 8192 pw_1697x16: times 2 dw 1697*16 pw_2896x8: times 2 dw 2896*8 pixel_10bpc_max: times 2 dw 0x03ff dconly_10bpc: times 2 dw 0x7c00 clip_18b_min: dd -0x20000 clip_18b_max: dd 0x1ffff pd_1: dd 1 pd_2: dd 2 pd_1448: dd 1448 pd_2048: dd 2048 pd_3071: dd 3071 ; 1024 + 2048 - 1 pd_3072: dd 3072 ; 1024 + 2048 pd_5119: dd 5119 ; 1024 + 4096 - 1 pd_5120: dd 5120 ; 1024 + 4096 pd_5793: dd 5793 cextern dup16_perm cextern int8_permA cextern idct64_mul_16bpc cextern idct_8x8_internal_8bpc_avx512icl.main cextern iadst_8x8_internal_8bpc_avx512icl.main_pass2 cextern idct_8x16_internal_8bpc_avx512icl.main cextern idct_8x16_internal_8bpc_avx512icl.main2 cextern idct_8x16_internal_8bpc_avx512icl.main_fast cextern idct_8x16_internal_8bpc_avx512icl.main_fast2 cextern iadst_8x16_internal_8bpc_avx512icl.main2 cextern idct_16x8_internal_8bpc_avx512icl.main cextern iadst_16x8_internal_8bpc_avx512icl.main_pass2 cextern idct_16x16_internal_8bpc_avx512icl.main cextern idct_16x16_internal_8bpc_avx512icl.main2 cextern idct_16x16_internal_8bpc_avx512icl.main_fast cextern idct_16x16_internal_8bpc_avx512icl.main_fast2 cextern iadst_16x16_internal_8bpc_avx512icl.main_pass2b cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_fast cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_fast2 cextern inv_txfm_add_dct_dct_8x32_8bpc_avx512icl.main_end cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf_fast cextern inv_txfm_add_dct_dct_16x32_8bpc_avx512icl.main_oddhalf_fast2 cextern inv_txfm_add_dct_dct_32x8_8bpc_avx512icl.main cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf_fast cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf_fast2 cextern inv_txfm_add_dct_dct_32x16_8bpc_avx512icl.main_oddhalf_fast3 cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf_fast cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf_fast2 cextern inv_txfm_add_dct_dct_32x32_8bpc_avx512icl.main_oddhalf_fast3 cextern inv_txfm_add_dct_dct_16x64_8bpc_avx512icl.main_oddhalf cextern inv_txfm_add_dct_dct_16x64_8bpc_avx512icl.main_oddhalf_fast cextern inv_txfm_add_dct_dct_32x64_8bpc_avx512icl.main_part1 cextern inv_txfm_add_dct_dct_32x64_8bpc_avx512icl.main_part1_fast cextern inv_txfm_add_dct_dct_32x64_8bpc_avx512icl.main_part1_fast2 cextern inv_txfm_add_dct_dct_32x64_8bpc_avx512icl.main_part2 SECTION .text %define o_base (pw_2048+4*128) %define o_base_8bpc (int8_permA+64*18) %define o(x) (r5 - o_base + (x)) %define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX) INIT_ZMM avx512icl ; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12 ; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12 ; flags: 1 = inv_dst1, 2 = inv_dst2 ; skip round/shift if rnd is not a number %macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], flags %if %8 < 32 pmulld m%4, m%1, m%8 pmulld m%3, m%2, m%8 %else %if %8 < 4096 vpbroadcastd m%3, [o(pd_%8)] %else vbroadcasti32x4 m%3, [o(pd_%8)] %endif pmulld m%4, m%1, m%3 pmulld m%3, m%2 %endif %if %7 < 32 pmulld m%1, m%7 pmulld m%2, m%7 %else %if %7 < 4096 vpbroadcastd m%5, [o(pd_%7)] %else vbroadcasti32x4 m%5, [o(pd_%7)] %endif pmulld m%1, m%5 pmulld m%2, m%5 %endif %if %9 & 2 psubd m%4, m%6, m%4 psubd m%2, m%4, m%2 %else %ifnum %6 paddd m%4, m%6 %endif paddd m%2, m%4 %endif %ifnum %6 paddd m%1, m%6 %endif %if %9 & 1 psubd m%1, m%3, m%1 %else psubd m%1, m%3 %endif %ifnum %6 psrad m%2, 12 psrad m%1, 12 %endif %endmacro %macro INV_TXFM_FN 4 ; type1, type2, eob_offset, size cglobal inv_txfm_add_%1_%2_%4_10bpc, 4, 7, 0, dst, stride, c, eob, tx2 %define %%p1 m(i%1_%4_internal_10bpc) lea r5, [o_base] ; Jump to the 1st txfm function if we're not taking the fast path, which ; in turn performs an indirect jump to the 2nd txfm function. lea tx2q, [m(i%2_%4_internal_10bpc).pass2] %ifidn %1_%2, dct_dct test eobd, eobd jnz %%p1 %else %if %3 add eobd, %3 %endif ; jump to the 1st txfm function unless it's located directly after this times ((%%end - %%p1) >> 31) & 1 jmp %%p1 ALIGN function_align %%end: %endif %endmacro %macro INV_TXFM_8X8_FN 2-3 0 ; type1, type2, eob_offset INV_TXFM_FN %1, %2, %3, 8x8 %ifidn %1_%2, dct_dct imul r6d, [cq], 181 mov [cq], eobd ; 0 or r3d, 8 .dconly: add r6d, 384 sar r6d, 9 .dconly2: vpbroadcastd ym2, [o(dconly_10bpc)] imul r6d, 181 add r6d, 2176 sar r6d, 12 vpbroadcastw ym1, r6d paddsw ym1, ym2 .dconly_loop: mova xm0, [dstq+strideq*0] vinserti32x4 ym0, [dstq+strideq*1], 1 paddsw ym0, ym1 psubusw ym0, ym2 mova [dstq+strideq*0], xm0 vextracti32x4 [dstq+strideq*1], ym0, 1 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop RET %endif %endmacro INV_TXFM_8X8_FN dct, dct INV_TXFM_8X8_FN dct, adst INV_TXFM_8X8_FN dct, flipadst INV_TXFM_8X8_FN dct, identity cglobal idct_8x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 call .load vpermi2q m1, m0, m2 ; 1 5 vpermi2q m3, m6, m4 ; 7 3 vpermt2q m0, m5, m4 ; 0 2 vpermt2q m2, m5, m6 ; 4 6 call .main call .main_end mova m4, [o(idct8x8p)] packssdw m0, m2 ; 0 1 4 5 packssdw m1, m3 ; 3 2 7 6 vpermb m0, m4, m0 vprolq m1, 32 vpermb m2, m4, m1 punpckhdq m1, m0, m2 punpckldq m0, m2 jmp tx2q .pass2: lea r5, [o_base_8bpc] vextracti32x8 ym2, m0, 1 vextracti32x8 ym3, m1, 1 call m(idct_8x8_internal_8bpc).main mova m10, [permC] vpbroadcastd m12, [pw_2048] .end: vpermt2q m0, m10, m1 vpermt2q m2, m10, m3 .end2: vpbroadcastd m11, [pixel_10bpc_max] lea r6, [strideq*3] pxor m10, m10 pmulhrsw m8, m12, m0 call .write_8x4_start pmulhrsw m8, m12, m2 .write_8x4: lea dstq, [dstq+strideq*4] add cq, 64*2 .write_8x4_start: mova xm9, [dstq+strideq*0] vinserti32x4 ym9, [dstq+strideq*1], 1 vinserti32x4 m9, [dstq+strideq*2], 2 vinserti32x4 m9, [dstq+r6 ], 3 mova [cq+64*0], m10 mova [cq+64*1], m10 paddw m9, m8 pmaxsw m9, m10 pminsw m9, m11 mova [dstq+strideq*0], xm9 vextracti32x4 [dstq+strideq*1], ym9, 1 vextracti32x4 [dstq+strideq*2], m9, 2 vextracti32x4 [dstq+r6 ], m9, 3 ret ALIGN function_align .load: mova m0, [cq+64*0] ; 0 1 mova m4, [cq+64*1] ; 2 3 mova m1, [o(permB)] mova m2, [cq+64*2] ; 4 5 mova m6, [cq+64*3] ; 6 7 vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] psrlq m5, m1, 32 vpbroadcastd m12, [o(pd_2896)] mova m3, m1 vpbroadcastd m11, [o(pd_1)] ret ALIGN function_align .main_fast: ; bottom half is zero vbroadcasti32x4 m3, [o(pd_4017_3406)] vbroadcasti32x4 m8, [o(pd_799_m2276)] vbroadcasti32x4 m2, [o(pd_2896_3784)] vbroadcasti32x4 m9, [o(pd_2896_1567)] pmulld m3, m1 ; t4a t5a pmulld m1, m8 ; t7a t6a pmulld m2, m0 ; t0 t3 pmulld m0, m9 ; t1 t2 jmp .main2 .main: ITX_MULSUB_2D 1, 3, 8, 9, 10, _, 799_3406, 4017_2276 ITX_MULSUB_2D 0, 2, 8, 9, 10, _, 2896_1567, 2896_3784 .main2: REPX {paddd x, m13}, m1, m3, m0, m2 REPX {psrad x, 12 }, m1, m3, m0, m2 punpcklqdq m8, m1, m3 ; t4a t7a punpckhqdq m1, m3 ; t5a t6a psubd m3, m8, m1 ; t5a t6a paddd m8, m1 ; t4 t7 pmaxsd m3, m14 punpckhqdq m1, m2, m0 ; t3 t2 pminsd m3, m15 punpcklqdq m2, m0 ; t0 t1 pmulld m3, m12 paddd m0, m2, m1 ; dct4 out0 out1 psubd m2, m1 ; dct4 out3 out2 REPX {pmaxsd x, m14}, m8, m0, m2 REPX {pminsd x, m15}, m8, m0, m2 .main3: pshufd m1, m3, q1032 paddd m3, m13 psubd m9, m3, m1 paddd m3, m1 psrad m9, 12 psrad m3, 12 punpckhqdq m1, m8, m3 ; t7 t6 shufpd m8, m9, 0xaa ; t4 t5 ret .main_end: paddd m0, m11 paddd m2, m11 psubd m3, m0, m1 ; out7 out6 paddd m0, m1 ; out0 out1 paddd m1, m2, m8 ; out3 out2 psubd m2, m8 ; out4 out5 REPX {vpsravd x, m11}, m0, m2, m3, m1 ret INV_TXFM_8X8_FN adst, dct INV_TXFM_8X8_FN adst, flipadst INV_TXFM_8X8_FN adst, identity INV_TXFM_8X8_FN adst, adst cglobal iadst_8x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 call m(idct_8x8_internal_10bpc).load vpermi2q m1, m6, m2 ; 7 5 vpermi2q m3, m4, m0 ; 3 1 vpermt2q m0, m5, m4 ; 0 2 vpermt2q m2, m5, m6 ; 4 6 call .main punpckldq m1, m2, m4 ; out4 out6 punpckhdq m2, m0 ; -out5 -out7 punpckldq m0, m3 ; out0 out2 punpckhdq m4, m3 ; -out1 -out3 paddd m1, m11 psubd m3, m11, m2 paddd m0, m11 psubd m4, m11, m4 .pass1_end: REPX {psrad x, 1}, m1, m0, m3, m4 packssdw m0, m1 ; 0 2 4 6 packssdw m4, m3 ; 1 3 5 7 psrlq m1, [o(permB)], 8 punpckhwd m3, m0, m4 punpcklwd m0, m4 psrlq m2, m1, 32 vpermi2q m1, m0, m3 vpermt2q m0, m2, m3 jmp tx2q .pass2: call .main_pass2 movu m10, [permC+2] vbroadcasti32x8 m12, [pw_2048_m2048+16] jmp m(idct_8x8_internal_10bpc).end .main_pass2: vextracti32x8 ym2, m0, 1 vextracti32x8 ym3, m1, 1 lea r5, [o_base_8bpc] pshufd ym4, ym0, q1032 pshufd ym5, ym1, q1032 jmp m(iadst_8x8_internal_8bpc).main_pass2 ALIGN function_align .main: ITX_MULSUB_2D 1, 0, 4, 5, 6, 13, 401_1931, 4076_3612 ITX_MULSUB_2D 3, 2, 4, 5, 6, 13, 3166_3920, 2598_1189 psubd m4, m0, m2 ; t4 t6 paddd m0, m2 ; t0 t2 psubd m2, m1, m3 ; t5 t7 paddd m1, m3 ; t1 t3 REPX {pmaxsd x, m14}, m4, m2, m0, m1 REPX {pminsd x, m15}, m4, m2, m0, m1 pxor m5, m5 psubd m5, m4 shufpd m4, m2, 0xaa ; t4 t7 shufpd m2, m5, 0xaa ; t5 -t6 ITX_MULSUB_2D 4, 2, 3, 5, 6, 13, 1567, 3784 punpckhqdq m3, m0, m1 punpcklqdq m0, m1 psubd m1, m0, m3 ; t2 t3 paddd m0, m3 ; out0 -out7 punpckhqdq m3, m4, m2 ; t7a t6a punpcklqdq m4, m2 ; t5a t4a psubd m2, m4, m3 ; t7 t6 paddd m4, m3 ; out6 -out1 REPX {pmaxsd x, m14}, m1, m2 REPX {pminsd x, m15}, m1, m2 shufpd m3, m1, m2, 0xaa shufpd m1, m2, 0x55 pmulld m3, m12 pmulld m1, m12 paddd m3, m13 psubd m2, m3, m1 paddd m3, m1 psrad m2, 12 ; out4 -out5 pshufd m3, m3, q1032 psrad m3, 12 ; out2 -out3 ret INV_TXFM_8X8_FN flipadst, dct INV_TXFM_8X8_FN flipadst, adst INV_TXFM_8X8_FN flipadst, identity INV_TXFM_8X8_FN flipadst, flipadst cglobal iflipadst_8x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 call m(idct_8x8_internal_10bpc).load vpermi2q m1, m6, m2 ; 7 5 vpermi2q m3, m4, m0 ; 3 1 vpermt2q m0, m5, m4 ; 0 2 vpermt2q m2, m5, m6 ; 4 6 call m(iadst_8x8_internal_10bpc).main punpckhdq m1, m3, m4 ; -out3 -out1 punpckldq m3, m0 ; out2 out0 punpckhdq m0, m2 ; -out7 -out5 punpckldq m4, m2 ; out6 out4 psubd m1, m11, m1 paddd m3, m11 psubd m0, m11, m0 paddd m4, m11 jmp m(iadst_8x8_internal_10bpc).pass1_end .pass2: call m(iadst_8x8_internal_10bpc).main_pass2 movu m10, [permC+1] vbroadcasti32x8 m12, [pw_m2048_2048+16] lea r6, [strideq*3] vpermt2q m0, m10, m1 ; 7 6 5 4 vpbroadcastd m11, [pixel_10bpc_max] vpermt2q m2, m10, m3 ; 3 2 1 0 pxor m10, m10 pmulhrsw m8, m12, m2 call m(idct_8x8_internal_10bpc).write_8x4_start pmulhrsw m8, m12, m0 jmp m(idct_8x8_internal_10bpc).write_8x4 INV_TXFM_8X8_FN identity, dct INV_TXFM_8X8_FN identity, adst INV_TXFM_8X8_FN identity, flipadst INV_TXFM_8X8_FN identity, identity cglobal iidentity_8x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 mova m1, [cq+64*0] packssdw m1, [cq+64*2] ; 0 4 1 5 mova m2, [cq+64*1] ; 2 6 3 7 packssdw m2, [cq+64*3] mova m0, [o(idtx8x8p)] vpermb m1, m0, m1 vpermb m2, m0, m2 punpckldq m0, m1, m2 ; 0 1 4 5 punpckhdq m1, m2 ; 2 3 6 7 jmp tx2q .pass2: movu m3, [o(permC+2)] vpbroadcastd m12, [o(pw_4096)] psrlq m2, m3, 32 vpermi2q m2, m0, m1 vpermt2q m0, m3, m1 jmp m(idct_8x8_internal_10bpc).end2 %macro INV_TXFM_8X16_FN 2-3 0 ; type1, type2, eob_offset INV_TXFM_FN %1, %2, %3, 8x16 %ifidn %1_%2, dct_dct imul r6d, [cq], 181 mov [cq], eobd ; 0 or r3d, 16 add r6d, 128 sar r6d, 8 imul r6d, 181 jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly %endif %endmacro INV_TXFM_8X16_FN dct, dct INV_TXFM_8X16_FN dct, identity, 35 INV_TXFM_8X16_FN dct, flipadst INV_TXFM_8X16_FN dct, adst cglobal idct_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp cmp eobd, 43 jl .fast call .load call .main call .main_end .pass1_end: packssdw m0, m4 packssdw m1, m5 packssdw m2, m6 packssdw m3, m7 jmp tx2q .pass2: mova m8, [o(idct8x16p)] REPX {vpermb x, m8, x}, m0, m1, m2, m3 punpckhdq m5, m0, m1 punpckldq m0, m1 punpckhdq m4, m2, m3 punpckldq m2, m3 punpcklqdq m8, m0, m2 ; 15 1 punpckhqdq m0, m2 ; 7 9 punpckhqdq m1, m5, m4 ; 3 13 punpcklqdq m5, m4 ; 11 5 lea r5, [o_base_8bpc] vextracti32x8 ym7, m8, 1 ; 14 2 vextracti32x8 ym3, m0, 1 ; 6 10 vextracti32x8 ym6, m1, 1 ; 12 4 vextracti32x8 ym9, m5, 1 ; 8 0 call m(idct_8x16_internal_8bpc).main2 mova m8, [permC] vpbroadcastd m12, [pw_2048] vpermt2q m0, m8, m1 lea r6, [strideq*3] vpermt2q m2, m8, m3 vpbroadcastd m11, [pixel_10bpc_max] vpermt2q m4, m8, m5 pxor m10, m10 vpermt2q m6, m8, m7 pmulhrsw m8, m12, m0 call m(idct_8x8_internal_10bpc).write_8x4_start pmulhrsw m8, m12, m2 call m(idct_8x8_internal_10bpc).write_8x4 pmulhrsw m8, m12, m4 call m(idct_8x8_internal_10bpc).write_8x4 pmulhrsw m8, m12, m6 jmp m(idct_8x8_internal_10bpc).write_8x4 .fast: mova ym0, [cq+64*0] mova ym4, [cq+64*2] mova ym1, [cq+64*1] mova ym5, [cq+64*5] mova ym2, [cq+64*4] mova ym6, [cq+64*6] mova ym3, [cq+64*7] mova ym7, [cq+64*3] call .round_input_fast call m(idct_8x8_internal_10bpc).main call m(idct_8x8_internal_10bpc).main_end movu m6, [o(permC+3)] packssdw m3, m1, m3 packssdw m1, m0, m2 vprolq m3, 32 vpermd m1, m6, m1 vpermd m3, m6, m3 mova ym0, ym1 ; 0 4 vextracti32x8 ym1, m1, 1 ; 1 5 mova ym2, ym3 ; 2 6 vextracti32x8 ym3, m3, 1 ; 3 7 jmp tx2q ALIGN function_align .round_input_fast: movshdup m8, [o(permB)] vpbroadcastd m12, [o(pd_2896)] vpermt2q m0, m8, m4 vpermt2q m1, m8, m5 vpermt2q m2, m8, m6 vpermt2q m3, m8, m7 vpbroadcastd m13, [o(pd_2048)] REPX {pmulld x, m12}, m0, m1, m2, m3 vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] REPX {paddd x, m13}, m0, m1, m2, m3 vpbroadcastd m11, [o(pd_1)] REPX {psrad x, 12 }, m0, m1, m2, m3 ret ALIGN function_align .load: vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] .load2: vpbroadcastd m12, [o(pd_2896)] pmulld m0, m12, [cq+64*0] pmulld m1, m12, [cq+64*1] pmulld m2, m12, [cq+64*2] pmulld m3, m12, [cq+64*3] vpbroadcastd m13, [o(pd_2048)] pmulld m4, m12, [cq+64*4] pmulld m5, m12, [cq+64*5] pmulld m6, m12, [cq+64*6] pmulld m7, m12, [cq+64*7] .round: REPX {paddd x, m13}, m0, m1, m2, m3 REPX {psrad x, 12 }, m0, m1, m2, m3 REPX {paddd x, m13}, m4, m5, m6, m7 REPX {psrad x, 12 }, m4, m5, m6, m7 ret ALIGN function_align .main_fast2_rect2: REPX {paddd x, m13}, m0, m1 REPX {psrad x, 12 }, m0, m1 .main_fast2: pmulld m0, m12 pmulld m6, m1, [o(pd_4017)] {1to16} ; t7a pmulld m8, m1, [o(pd_799)] {1to16} ; t4a REPX {paddd x, m13}, m0, m6, m8 REPX {psrad x, 12 }, m0, m6, m8 pmulld m5, m6, m12 pmulld m1, m8, m12 paddd m5, m13 psubd m4, m5, m1 paddd m5, m1 REPX {psrad x, 12 }, m4, m5 REPX {mova x, m0 }, m1, m2, m3 ret .main_fast_rect2: REPX {paddd x, m13}, m0, m1, m2, m3 REPX {psrad x, 12 }, m0, m1, m2, m3 .main_fast: pmulld m0, m12 pmulld m5, m3, [o(pd_2276)] {1to16} ; t5a pmulld m3, [o(pd_3406)] {1to16} ; t6a pmulld m7, m1, [o(pd_4017)] {1to16} ; t7a pmulld m1, [o(pd_799)] {1to16} ; t4a pmulld m6, m2, [o(pd_3784)] {1to16} ; t3 pmulld m2, [o(pd_1567)] {1to16} ; t2 paddd m0, m13 psubd m5, m13, m5 psrad m0, 12 ; t0 mova m9, m0 ; t1 jmp .main2 .main_rect2: call .round .main: pmulld m0, m12 ITX_MULSUB_2D 5, 3, 8, 9, 10, _, 3406, 2276 ; t5a t6a ITX_MULSUB_2D 1, 7, 8, 9, 10, _, 799, 4017 ; t4a t7a ITX_MULSUB_2D 2, 6, 8, 9, 10, _, 1567, 3784 ; t2 t3 pmulld m4, m12 paddd m0, m13 paddd m5, m13 psubd m9, m0, m4 ; t1 paddd m0, m4 ; t0 psrad m9, 12 psrad m0, 12 .main2: REPX {paddd x, m13}, m3, m1, m7 REPX {psrad x, 12 }, m5, m1, m3, m7 paddd m8, m1, m5 ; t4 psubd m1, m5 ; t5a psubd m5, m7, m3 ; t6a paddd m7, m3 ; t7 pmaxsd m5, m14 pmaxsd m1, m14 paddd m2, m13 paddd m6, m13 pminsd m5, m15 pminsd m1, m15 pmulld m5, m12 pmulld m1, m12 pmaxsd m8, m14 pmaxsd m7, m14 pminsd m8, m15 paddd m5, m13 psubd m4, m5, m1 paddd m5, m1 REPX {psrad x, 12 }, m2, m6, m5, m4 paddd m1, m9, m2 ; dct4 out1 psubd m2, m9, m2 ; dct4 out2 psubd m3, m0, m6 ; dct4 out3 paddd m0, m6 ; dct4 out0 pminsd m6, m15, m7 REPX {pmaxsd x, m14}, m0, m1, m2, m3 REPX {pminsd x, m15}, m0, m1, m2, m3 ret .main_end: vpbroadcastd m11, [o(pd_1)] .main_end2: REPX {paddd x, m11}, m0, m1, m2, m3 psubd m7, m0, m6 ; out7 paddd m0, m6 ; out0 psubd m6, m1, m5 ; out6 paddd m1, m5 ; out1 psubd m5, m2, m4 ; out5 paddd m2, m4 ; out2 psubd m4, m3, m8 ; out4 paddd m3, m8 ; out3 REPX {vpsravd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 ret INV_TXFM_8X16_FN adst, dct INV_TXFM_8X16_FN adst, identity, 35 INV_TXFM_8X16_FN adst, flipadst INV_TXFM_8X16_FN adst, adst cglobal iadst_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp cmp eobd, 43 jl .fast call m(idct_8x16_internal_10bpc).load call .main psrad m0, 1 psrad m1, 1 psrad m6, m10, 1 psrad m7, m11, 1 psrad m2, 12 psrad m3, 12 psrad m4, m8, 12 psrad m5, m9, 12 jmp m(idct_8x16_internal_10bpc).pass1_end .fast: call .fast_main punpcklqdq m1, m2, m4 ; out4 out6 punpckhqdq m2, m0 ; -out5 -out7 punpcklqdq m0, m3 ; out0 out2 punpckhqdq m4, m3 ; -out1 -out3 paddd m1, m11 psubd m3, m11, m2 paddd m0, m11 psubd m4, m11, m4 .fast_end: movu m5, [o(permC+3)] REPX {psrad x, 1}, m1, m0, m3, m4 packssdw m2, m0, m1 ; 0 2 4 6 packssdw m3, m4, m3 ; 1 3 5 7 vpermd m2, m5, m2 vpermd m3, m5, m3 mova ym0, ym2 vextracti32x8 ym2, m2, 1 mova ym1, ym3 vextracti32x8 ym3, m3, 1 jmp tx2q .pass2: call .pass2_main movu m4, [permB+2] vbroadcasti32x8 m12, [pw_2048_m2048+16] psrlq m7, m4, 8 vpermi2q m4, m0, m3 ; 0 1 2 3 psrlq m5, m7, 24 vpermi2q m7, m0, m3 ; 12 13 14 15 psrlq m6, m5, 8 vpermq m5, m5, m1 ; 4 5 6 7 vpermq m6, m6, m2 ; 8 9 10 11 .pass2_end: vpbroadcastd m11, [pixel_10bpc_max] pxor m10, m10 lea r6, [strideq*3] pmulhrsw m8, m12, m4 call m(idct_8x8_internal_10bpc).write_8x4_start pmulhrsw m8, m12, m5 call m(idct_8x8_internal_10bpc).write_8x4 pmulhrsw m8, m12, m6 call m(idct_8x8_internal_10bpc).write_8x4 pmulhrsw m8, m12, m7 jmp m(idct_8x8_internal_10bpc).write_8x4 ALIGN function_align .main: ITX_MULSUB_2D 7, 0, 8, 9, 10, 13, 401, 4076 ; t1a, t0a ITX_MULSUB_2D 1, 6, 8, 9, 10, 13, 3920, 1189 ; t7a, t6a ITX_MULSUB_2D 5, 2, 8, 9, 10, 13, 1931, 3612 ; t3a, t2a ITX_MULSUB_2D 3, 4, 8, 9, 10, 13, 3166, 2598 ; t5a, t4a psubd m8, m2, m6 ; t6 paddd m2, m6 ; t2 psubd m6, m0, m4 ; t4 paddd m0, m4 ; t0 psubd m4, m5, m1 ; t7 paddd m5, m1 ; t3 psubd m1, m7, m3 ; t5 paddd m7, m3 ; t1 REPX {pmaxsd x, m14}, m6, m1, m8, m4, m2, m0, m5, m7 REPX {pminsd x, m15}, m6, m1, m8, m4, m2, m0, m5, m7 vpbroadcastd m10, [o(pd_1567)] vpbroadcastd m11, [o(pd_3784)] ITX_MULSUB_2D 6, 1, 3, 9, _, 13, 10, 11 ; t5a, t4a ITX_MULSUB_2D 4, 8, 3, 9, _, 13, 11, 10 ; t6a, t7a vpbroadcastd m12, [o(pd_1448)] psubd m9, m6, m8 ; t7 paddd m6, m8 ; out6 psubd m3, m7, m5 ; t3 paddd m7, m5 ; -out7 psubd m5, m0, m2 ; t2 paddd m0, m2 ; out0 psubd m2, m1, m4 ; t6 paddd m1, m4 ; -out1 REPX {pmaxsd x, m14}, m5, m3, m2, m9 REPX {pminsd x, m15}, m5, m3, m2, m9 REPX {pmulld x, m12}, m5, m3, m2, m9 vpbroadcastd m4, [o(pd_1)] psubd m8, m5, m3 ; (t2 - t3) * 1448 paddd m3, m5 ; (t2 + t3) * 1448 psubd m5, m2, m9 ; (t6 - t7) * 1448 paddd m2, m9 ; (t6 + t7) * 1448 vpbroadcastd m9, [o(pd_3072)] paddd m0, m4 psubd m1, m4, m1 paddd m10, m6, m4 psubd m11, m4, m7 paddd m2, m9 paddd m8, m9 vpbroadcastd m9, [o(pd_3071)] psubd m3, m9, m3 psubd m9, m5 ret ALIGN function_align .fast_main: mova ym0, [cq+64*0] mova ym4, [cq+64*2] mova ym1, [cq+64*7] mova ym5, [cq+64*5] mova ym2, [cq+64*4] mova ym6, [cq+64*6] mova ym3, [cq+64*3] mova ym7, [cq+64*1] call m(idct_8x16_internal_10bpc).round_input_fast jmp m(iadst_8x8_internal_10bpc).main ALIGN function_align .pass2_main: mova m8, [o(iadst8x16p)] REPX {vpermb x, m8, x}, m0, m1, m2, m3 vpbroadcastd m10, [o(pw_2896x8)] punpckhdq m5, m0, m1 punpckldq m0, m1 punpckhdq m1, m2, m3 punpckldq m2, m3 lea r5, [o_base_8bpc] punpckhqdq m4, m0, m2 ; 12 3 14 1 punpcklqdq m0, m2 ; 0 15 2 13 punpckhqdq m6, m5, m1 ; 8 7 10 5 punpcklqdq m5, m1 ; 4 11 6 9 call m(iadst_8x16_internal_8bpc).main2 paddsw m1, m2, m4 psubsw m2, m4 pmulhrsw m1, m10 ; -out7 out4 out6 -out5 pmulhrsw m2, m10 ; out8 -out11 -out9 out10 ret INV_TXFM_8X16_FN flipadst, dct INV_TXFM_8X16_FN flipadst, identity, 35 INV_TXFM_8X16_FN flipadst, adst INV_TXFM_8X16_FN flipadst, flipadst cglobal iflipadst_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp cmp eobd, 43 jl .fast call m(idct_8x16_internal_10bpc).load call m(iadst_8x16_internal_10bpc).main psrad m7, m0, 1 psrad m0, m11, 1 psrad m6, m1, 1 psrad m1, m10, 1 psrad m5, m2, 12 psrad m2, m9, 12 psrad m4, m3, 12 psrad m3, m8, 12 jmp m(idct_8x16_internal_10bpc).pass1_end .fast: call m(iadst_8x16_internal_10bpc).fast_main punpckhqdq m1, m3, m4 ; -out3 -out1 punpcklqdq m3, m0 ; out2 out0 punpckhqdq m0, m2 ; -out7 -out5 punpcklqdq m4, m2 ; out6 out4 psubd m1, m11, m1 paddd m3, m11 psubd m0, m11, m0 paddd m4, m11 jmp m(iadst_8x16_internal_10bpc).fast_end .pass2: call m(iadst_8x16_internal_10bpc).pass2_main movu m7, [permB+2] vbroadcasti32x8 m12, [pw_m2048_2048+16] psrlq m4, m7, 8 vpermi2q m7, m3, m0 ; 3 2 1 0 psrlq m5, m4, 24 vpermi2q m4, m3, m0 ; 15 14 13 12 psrlq m6, m5, 8 vpermq m5, m5, m2 ; 11 10 9 8 vpermq m6, m6, m1 ; 7 6 5 4 jmp m(iadst_8x16_internal_10bpc).pass2_end INV_TXFM_8X16_FN identity, dct INV_TXFM_8X16_FN identity, adst INV_TXFM_8X16_FN identity, flipadst INV_TXFM_8X16_FN identity, identity cglobal iidentity_8x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 call m(idct_8x16_internal_10bpc).load2 jmp m(idct_8x16_internal_10bpc).pass1_end .pass2: vpbroadcastd m8, [o(pw_1697x16)] pmulhrsw m4, m8, m0 pmulhrsw m5, m8, m1 pmulhrsw m6, m8, m2 pmulhrsw m7, m8, m3 REPX {paddsw x, x}, m0, m1, m2, m3 paddsw m0, m4 paddsw m1, m5 paddsw m2, m6 paddsw m3, m7 vpbroadcastd m7, [o(pw_2048)] punpckhwd m4, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 punpcklwd m2, m3 vpbroadcastd m6, [o(pixel_10bpc_max)] punpckhdq m3, m0, m2 punpckldq m0, m2 punpckldq m2, m4, m1 punpckhdq m4, m1 pxor m5, m5 punpckhqdq m1, m0, m2 ; 1 5 9 13 punpcklqdq m0, m2 ; 0 4 8 12 punpcklqdq m2, m3, m4 ; 2 6 10 14 punpckhqdq m3, m4 ; 3 7 11 15 lea r6, [strideq*3] pmulhrsw m0, m7 call .write_8x4_start pmulhrsw m0, m7, m1 call .write_8x4 pmulhrsw m0, m7, m2 call .write_8x4 pmulhrsw m0, m7, m3 .write_8x4: add dstq, strideq add cq, 64*2 .write_8x4_start: mova xm4, [dstq+strideq*0] vinserti32x4 ym4, [dstq+strideq*4], 1 vinserti32x4 m4, [dstq+strideq*8], 2 vinserti32x4 m4, [dstq+r6*4 ], 3 mova [cq+64*0], m5 mova [cq+64*1], m5 paddw m4, m0 pmaxsw m4, m5 pminsw m4, m6 mova [dstq+strideq*0], xm4 vextracti32x4 [dstq+strideq*4], ym4, 1 vextracti32x4 [dstq+strideq*8], m4, 2 vextracti32x4 [dstq+r6*4 ], m4, 3 ret %macro INV_TXFM_16X8_FN 2-3 0 ; type1, type2, eob_offset INV_TXFM_FN %1, %2, %3, 16x8 %ifidn %1_%2, dct_dct imul r6d, [cq], 181 mov [cq], eobd ; 0 or r3d, 8 .dconly: add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 384 sar r6d, 9 .dconly2: vpbroadcastd m2, [o(dconly_10bpc)] imul r6d, 181 add r6d, 2176 sar r6d, 12 vpbroadcastw m1, r6d paddsw m1, m2 .dconly_loop: mova ym0, [dstq+strideq*0] vinserti32x8 m0, [dstq+strideq*1], 1 paddsw m0, m1 psubusw m0, m2 mova [dstq+strideq*0], ym0 vextracti32x8 [dstq+strideq*1], m0, 1 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop RET %endif %endmacro INV_TXFM_16X8_FN dct, dct INV_TXFM_16X8_FN dct, identity, -21 INV_TXFM_16X8_FN dct, flipadst INV_TXFM_16X8_FN dct, adst cglobal idct_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp vpbroadcastd m12, [o(pd_2896)] pmulld m4, m12, [cq+64*0] ; 0 1 pmulld m9, m12, [cq+64*1] ; 2 3 pmulld m8, m12, [cq+64*2] ; 4 5 pmulld m7, m12, [cq+64*3] ; 6 7 vpbroadcastd m13, [o(pd_2048)] pxor m2, m2 mova m15, [o(permB)] REPX {mova [cq+64*x], m2}, 0, 1, 2, 3 psrlq m0, m15, 32 REPX {paddd x, m13}, m4, m9, m8, m7 vpbroadcastd m14, [o(clip_18b_min)] REPX {psrad x, 12 }, m4, m8, m9, m7 mova m1, m0 vpermi2q m0, m4, m8 ; 0 4 cmp eobd, 43 jl .fast pmulld m5, m12, [cq+64*4] ; 8 9 pmulld m10, m12, [cq+64*5] ; 10 11 pmulld m11, m12, [cq+64*6] ; 12 13 pmulld m6, m12, [cq+64*7] ; 14 15 REPX {mova [cq+64*x], m2}, 4, 5, 6, 7 REPX {paddd x, m13}, m5, m10, m11, m6 REPX {psrad x, 12 }, m10, m5, m11, m6 mova m2, m1 vpermi2q m1, m9, m10 ; 2 10 mova m3, m2 vpermi2q m2, m5, m11 ; 8 12 vpermi2q m3, m6, m7 ; 14 6 vpermt2q m4, m15, m11 ; 1 13 vpermt2q m6, m15, m9 ; 15 3 vpermt2q m5, m15, m8 ; 9 5 vpermt2q m7, m15, m10 ; 7 11 vpbroadcastd m15, [o(clip_18b_max)] call m(idct_8x8_internal_10bpc).main call .main jmp .pass1_end .fast: vpermi2q m1, m9, m7 ; 2 6 vpermt2q m4, m15, m9 ; 1 3 vpermt2q m7, m15, m8 ; 7 5 vpbroadcastd m15, [o(clip_18b_max)] call m(idct_8x8_internal_10bpc).main_fast call .main_fast .pass1_end: call m(idct_8x16_internal_10bpc).main_end mova m8, [o(permA)] psrlq m9, m8, 8 .pass1_end2: mova m10, m9 mova m11, m8 call .transpose_16x8 jmp tx2q .pass2: lea r5, [o_base_8bpc] call m(idct_16x8_internal_8bpc).main movshdup m4, [permC] vpbroadcastd m11, [pw_2048] psrlq m5, m4, 8 .end: vpbroadcastd m13, [pixel_10bpc_max] pxor m12, m12 vpermq m8, m4, m0 vpermq m9, m5, m1 lea r6, [strideq*3] call .write_16x4 vpermq m8, m4, m2 vpermq m9, m5, m3 .write_16x4: pmulhrsw m8, m11 pmulhrsw m9, m11 .write_16x4_noround: mova ym10, [dstq+strideq*0] vinserti32x8 m10, [dstq+strideq*1], 1 paddw m8, m10 mova ym10, [dstq+strideq*2] vinserti32x8 m10, [dstq+r6 ], 1 paddw m9, m10 pmaxsw m8, m12 pmaxsw m9, m12 pminsw m8, m13 pminsw m9, m13 mova [dstq+strideq*0], ym8 vextracti32x8 [dstq+strideq*1], m8, 1 mova [dstq+strideq*2], ym9 vextracti32x8 [dstq+r6 ], m9, 1 lea dstq, [dstq+strideq*4] ret ALIGN function_align .main_fast: ; bottom half is zero vbroadcasti32x4 m6, [o(pd_4076_3920)] vbroadcasti32x4 m3, [o(pd_401_m1189)] vbroadcasti32x4 m5, [o(pd_m2598_1931)] vbroadcasti32x4 m9, [o(pd_3166_3612)] pmulld m6, m4 ; t15a t12a pmulld m4, m3 ; t8a t11a pmulld m5, m7 ; t9a t10a pmulld m7, m9 ; t14a t13a jmp .main2 .main: ITX_MULSUB_2D 4, 6, 3, 9, 10, _, 401_3920, 4076_1189 ITX_MULSUB_2D 5, 7, 3, 9, 10, _, 3166_1931, 2598_3612 .main2: REPX {paddd x, m13}, m4, m6, m5, m7 REPX {psrad x, 12 }, m4, m5, m6, m7 paddd m9, m4, m5 ; t8 t11 psubd m4, m5 ; t9 t10 psubd m5, m6, m7 ; t14 t13 paddd m6, m7 ; t15 t12 REPX {pmaxsd x, m14}, m5, m4, m9, m6 REPX {pminsd x, m15}, m5, m4, m9, m6 .main3: psubd m3, m0, m1 ; dct8 out7 out6 paddd m0, m1 ; dct8 out0 out1 vbroadcasti32x4 m7, [o(pd_3784_m3784)] pmulld m7, m5 vpmulld m5, [o(pd_1567)] {1to16} paddd m1, m2, m8 ; dct8 out3 out2 psubd m2, m8 ; dct8 out4 out5 vbroadcasti32x4 m8, [o(pd_1567_m1567)] pmulld m8, m4 vpmulld m4, [o(pd_3784)] {1to16} REPX {pmaxsd x, m14}, m0, m1 REPX {pminsd x, m15}, m0, m1 paddd m7, m13 paddd m5, m13 paddd m7, m8 psubd m5, m4 psrad m7, 12 ; t14a t10a psrad m5, 12 ; t9a t13a punpckhqdq m4, m9, m7 punpcklqdq m8, m9, m5 punpckhqdq m5, m6, m5 punpcklqdq m6, m7 psubd m7, m8, m4 ; t11a t10 paddd m8, m4 ; t8a t9 psubd m4, m6, m5 ; t12a t13 paddd m6, m5 ; t15a t14 REPX {pmaxsd x, m14}, m4, m7 REPX {pminsd x, m15}, m4, m7 pmulld m4, m12 pmulld m7, m12 REPX {pmaxsd x, m14}, m2, m3, m6, m8 REPX {pminsd x, m15}, m2, m3, m6, m8 paddd m4, m13 paddd m5, m4, m7 psubd m4, m7 psrad m4, 12 ; t11 t10a psrad m5, 12 ; t12 t13a ret ALIGN function_align .transpose_16x8: packssdw m0, m4 packssdw m1, m5 packssdw m2, m6 packssdw m3, m7 vpermi2d m8, m0, m2 vpermt2d m0, m9, m2 vpermi2d m10, m1, m3 vpermi2d m11, m1, m3 punpckhwd m3, m8, m0 punpcklwd m1, m8, m0 punpckhwd m4, m10, m11 punpcklwd m2, m10, m11 punpckldq m0, m1, m2 punpckhdq m1, m2 punpckldq m2, m3, m4 punpckhdq m3, m4 ret INV_TXFM_16X8_FN adst, dct INV_TXFM_16X8_FN adst, identity, -21 INV_TXFM_16X8_FN adst, flipadst INV_TXFM_16X8_FN adst, adst cglobal iadst_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp call .main_pass1 vpbroadcastd m9, [o(pd_1)] paddd m0, m9 psubd m1, m9, m1 paddd m2, m9 psubd m3, m9, m3 paddd m4, m9, m5 psubd m5, m9, m6 paddd m6, m9, m7 psubd m7, m9, m8 .pass1_end: mova m9, [o(permA)] psrlq m8, m9, 8 REPX {psrad x, 1}, m0, m4, m1, m5, m2, m6, m3, m7 jmp m(idct_16x8_internal_10bpc).pass1_end2 .pass2: call .main_pass2 vpermq m8, m11, m0 vpermq m9, m11, m1 call m(idct_16x8_internal_10bpc).write_16x4_noround vpermq m8, m11, m2 vpermq m9, m11, m3 jmp m(idct_16x8_internal_10bpc).write_16x4_noround ALIGN function_align .main_pass1: vpbroadcastd m12, [o(pd_2896)] pmulld m2, m12, [cq+64*0] pmulld m7, m12, [cq+64*1] pmulld m1, m12, [cq+64*2] pmulld m5, m12, [cq+64*3] vpbroadcastd m13, [o(pd_2048)] pxor m4, m4 mova m10, [o(permB)] REPX {mova [cq+64*x], m4}, 0, 1, 2, 3 REPX {paddd x, m13}, m2, m7, m1, m5 psrlq m6, m10, 32 REPX {psrad x, 12 }, m2, m7, m1, m5 mova m0, m6 vpermi2q m0, m2, m7 ; 0 2 vpermt2q m7, m10, m2 ; 3 1 mova m2, m6 vpermi2q m2, m1, m5 ; 4 6 vpermt2q m5, m10, m1 ; 7 5 cmp eobd, 43 jl .main_fast pmulld m8, m12, [cq+64*4] pmulld m3, m12, [cq+64*5] pmulld m9, m12, [cq+64*6] pmulld m1, m12, [cq+64*7] REPX {mova [cq+64*x], m4}, 4, 5, 6, 7 REPX {paddd x, m13}, m8, m3, m9, m1 REPX {psrad x, 12 }, m8, m3, m9, m1 mova m4, m6 vpermi2q m4, m8, m3 ; 8 10 vpermt2q m3, m10, m8 ; 11 9 vpermi2q m6, m9, m1 ; 12 14 vpermt2q m1, m10, m9 ; 15 13 .main: ITX_MULSUB_2D 1, 0, 8, 9, 10, _, 201_995, 4091_3973, 1 ITX_MULSUB_2D 3, 2, 8, 9, 10, _, 1751_2440, 3703_3290, 1 ITX_MULSUB_2D 5, 4, 8, 9, 10, _, 3035_3513, 2751_2106 ITX_MULSUB_2D 7, 6, 8, 9, 10, _, 3857_4052, 1380_601 jmp .main2 .main_fast: vbroadcasti32x4 m1, [o(pd_4091_3973)] vbroadcasti32x4 m8, [o(pd_201_995)] vbroadcasti32x4 m3, [o(pd_3703_3290)] vbroadcasti32x4 m9, [o(pd_1751_2440)] vbroadcasti32x4 m4, [o(pd_2751_2106)] vbroadcasti32x4 m10, [o(pd_3035_3513)] vbroadcasti32x4 m6, [o(pd_1380_601)] vbroadcasti32x4 m11, [o(pd_3857_4052)] pmulld m1, m0 pmulld m0, m8 pmulld m3, m2 pmulld m2, m9 pmulld m4, m5 pmulld m5, m10 pmulld m6, m7 pmulld m7, m11 .main2: vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] REPX {psubd x, m13, x}, m1, m3 REPX {paddd x, m13 }, m0, m2, m4, m5, m6, m7 REPX {psrad x, 12 }, m0, m4, m1, m5, m2, m6, m3, m7 psubd m8, m0, m4 ; t8a t10a paddd m0, m4 ; t0a t2a psubd m4, m1, m5 ; t9a t11a paddd m1, m5 ; t1a t3a psubd m5, m2, m6 ; t12a t14a paddd m2, m6 ; t4a t6a psubd m6, m3, m7 ; t13a t15a paddd m3, m7 ; t5a t7a REPX {pmaxsd x, m14}, m8, m4, m5, m6 REPX {pminsd x, m15}, m8, m4, m5, m6 vbroadcasti32x4 m11, [o(pd_4017_2276)] vbroadcasti32x4 m10, [o(pd_799_3406)] ITX_MULSUB_2D 8, 4, 7, 9, _, 13, 10, 11 ITX_MULSUB_2D 6, 5, 7, 9, _, 13, 11, 10 REPX {pmaxsd x, m14}, m0, m2, m1, m3 REPX {pminsd x, m15}, m0, m2, m1, m3 psubd m7, m0, m2 ; t4 t6 paddd m0, m2 ; t0 t2 psubd m2, m1, m3 ; t5 t7 paddd m1, m3 ; t1 t3 psubd m3, m4, m6 ; t12a t14a paddd m4, m6 ; t8a t10a psubd m6, m8, m5 ; t13a t15a paddd m8, m5 ; t9a t11a REPX {pmaxsd x, m14}, m7, m3, m2, m6 REPX {pminsd x, m15}, m7, m3, m2, m6 punpcklqdq m5, m3, m7 ; t12a t4 punpckhqdq m3, m7 ; t14a t6 punpckhqdq m7, m6, m2 ; t15a t7 punpcklqdq m6, m2 ; t13a t5 vpbroadcastd m11, [o(pd_1567)] vpbroadcastd m10, [o(pd_3784)] ITX_MULSUB_2D 7, 3, 2, 9, 10, 13, 10, 11 ITX_MULSUB_2D 5, 6, 2, 9, 10, 13, 11, 10 REPX {pmaxsd x, m14}, m0, m4, m1, m8 REPX {pminsd x, m15}, m0, m4, m1, m8 punpckhqdq m2, m4, m0 ; t10a t2 punpcklqdq m4, m0 ; t8a t0 punpckhqdq m0, m8, m1 ; t11a t3 punpcklqdq m8, m1 ; t9a t1 paddd m1, m6, m7 ; out2 -out3 psubd m6, m7 ; t14a t6 paddd m7, m5, m3 ; -out13 out12 psubd m5, m3 ; t15a t7 psubd m3, m8, m0 ; t11 t3a paddd m8, m0 ; out14 -out15 paddd m0, m4, m2 ; -out1 out0 psubd m4, m2 ; t10 t2a REPX {pmaxsd x, m14}, m6, m5, m3, m4 mov r6d, 0x3333 REPX {pminsd x, m15}, m6, m5, m3, m4 kmovw k1, r6d REPX {pmulld x, m12}, m6, m5, m3, m4 pxor m9, m9 REPX {vpsubd x{k1}, m9, x}, m0, m1, m7, m8 paddd m6, m13 paddd m4, m13 paddd m2, m6, m5 ; -out5 out4 psubd m6, m5 ; out10 -out11 psubd m5, m4, m3 ; -out9 out8 paddd m3, m4 ; out6 -out7 REPX {psrad x, 12}, m2, m3, m5, m6 REPX {vpsubd x{k1}, m9, x}, m2, m3, m5, m6 ret ALIGN function_align .main_pass2: lea r5, [o_base_8bpc] pshufd m4, m0, q1032 pshufd m5, m1, q1032 call m(iadst_16x8_internal_8bpc).main_pass2 movshdup m11, [permC] pmulhrsw m0, m6 pmulhrsw m1, m6 vpbroadcastd m13, [pixel_10bpc_max] pxor m12, m12 lea r6, [strideq*3] ret INV_TXFM_16X8_FN flipadst, dct INV_TXFM_16X8_FN flipadst, identity, -21 INV_TXFM_16X8_FN flipadst, adst INV_TXFM_16X8_FN flipadst, flipadst cglobal iflipadst_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 call m(iadst_16x8_internal_10bpc).main_pass1 vpbroadcastd m9, [o(pd_1)] psubd m4, m9, m3 paddd m3, m9, m5 paddd m5, m9, m2 psubd m2, m9, m6 psubd m6, m9, m1 paddd m1, m9, m7 paddd m7, m9, m0 psubd m0, m9, m8 jmp m(iadst_16x8_internal_10bpc).pass1_end .pass2: call m(iadst_16x8_internal_10bpc).main_pass2 psrlq m11, 8 vpermq m8, m11, m3 vpermq m9, m11, m2 call m(idct_16x8_internal_10bpc).write_16x4_noround vpermq m8, m11, m1 vpermq m9, m11, m0 jmp m(idct_16x8_internal_10bpc).write_16x4_noround INV_TXFM_16X8_FN identity, dct INV_TXFM_16X8_FN identity, adst INV_TXFM_16X8_FN identity, flipadst INV_TXFM_16X8_FN identity, identity cglobal iidentity_16x8_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 call m(idct_8x16_internal_10bpc).load2 vpbroadcastd m8, [o(pd_5793)] vpbroadcastd m13, [o(pd_3072)] pxor m10, m10 REPX {pmulld x, m8}, m0, m1, m2, m3, m4, m5, m6, m7 REPX {mova [cq+64*x], m10}, 0, 1, 2, 3, 4, 5, 6, 7 call m(idct_8x16_internal_10bpc).round psrlq m8, [o(permA)], 16 psrlq m9, m8, 8 mova m10, m8 mova m11, m9 call m(idct_16x8_internal_10bpc).transpose_16x8 jmp tx2q .pass2: movshdup m4, [o(permC)] vpbroadcastd m11, [o(pw_4096)] mova m5, m4 jmp m(idct_16x8_internal_10bpc).end %macro INV_TXFM_16X16_FN 2-3 0 ; type1, type2, eob_offset INV_TXFM_FN %1, %2, %3, 16x16 %ifidn %1_%2, dct_dct imul r6d, [cq], 181 mov [cq], eobd ; 0 or r3d, 16 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly2 %endif %endmacro INV_TXFM_16X16_FN dct, dct INV_TXFM_16X16_FN dct, identity, 28 INV_TXFM_16X16_FN dct, flipadst INV_TXFM_16X16_FN dct, adst cglobal idct_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] cmp eobd, 36 jl .fast mova m0, [cq+64* 0] mova m1, [cq+64* 2] mova m2, [cq+64* 4] mova m3, [cq+64* 6] mova m4, [cq+64* 8] mova m5, [cq+64*10] mova m6, [cq+64*12] mova m7, [cq+64*14] %if WIN64 movaps [cq+16*0], xmm6 movaps [cq+16*1], xmm7 %endif call m(idct_8x16_internal_10bpc).main mova m16, [cq+64* 1] mova m17, [cq+64* 3] mova m18, [cq+64* 5] mova m19, [cq+64* 7] mova m20, [cq+64* 9] mova m21, [cq+64*11] mova m22, [cq+64*13] mova m23, [cq+64*15] call .main call .main_end .pass1_end: %if WIN64 movaps xmm6, [cq+16*0] movaps xmm7, [cq+16*1] %endif vzeroupper .pass1_end2: call .main_end3 .pass1_end3: mov r6d, 64*12 pxor m8, m8 .zero_loop: mova [cq+r6+64*3], m8 mova [cq+r6+64*2], m8 mova [cq+r6+64*1], m8 mova [cq+r6+64*0], m8 sub r6d, 64*4 jge .zero_loop jmp tx2q .pass2: lea r5, [o_base_8bpc] call m(idct_16x16_internal_8bpc).main movshdup m12, [permC] vpbroadcastd m11, [pw_2048] psrlq m13, m12, 8 vpermq m8, m12, m0 vpermq m0, m13, m7 vpermq m7, m13, m1 vpermq m1, m12, m6 vpermq m6, m12, m2 vpermq m2, m13, m5 vpermq m5, m13, m3 vpermq m3, m12, m4 .pass2_end: lea r6, [strideq*3] vpbroadcastd m13, [pixel_10bpc_max] pxor m12, m12 pmulhrsw m8, m11, m8 pmulhrsw m9, m11, m7 call m(idct_16x8_internal_10bpc).write_16x4_noround pmulhrsw m8, m11, m6 pmulhrsw m9, m11, m5 call m(idct_16x8_internal_10bpc).write_16x4_noround pmulhrsw m8, m11, m3 pmulhrsw m9, m11, m2 call m(idct_16x8_internal_10bpc).write_16x4_noround pmulhrsw m8, m11, m1 pmulhrsw m9, m11, m0 jmp m(idct_16x8_internal_10bpc).write_16x4_noround .fast: mova ym0, [cq+64*0] mova ym2, [cq+64*4] movshdup m8, [o(permB)] mova ym1, [cq+64*2] mova ym3, [cq+64*6] mova ym4, [cq+64*1] mova ym5, [cq+64*3] mova ym6, [cq+64*5] mova ym7, [cq+64*7] vpermt2q m0, m8, m2 ; 0 4 vpermt2q m1, m8, m3 ; 2 6 vpermt2q m4, m8, m5 ; 1 3 vpermt2q m7, m8, m6 ; 7 5 call m(idct_8x8_internal_10bpc).main_fast call m(idct_16x8_internal_10bpc).main_fast vpbroadcastd m11, [o(pd_2)] call m(idct_8x16_internal_10bpc).main_end2 mova m8, [o(permA)] psrlq m9, m8, 8 jmp m(iadst_16x16_internal_10bpc).pass1_fast_end2 ALIGN function_align .main_fast2_rect2: REPX {paddd x, m13}, m16, m17 REPX {psrad x, 12 }, m16, m17 .main_fast2: pmulld m22, m16, [o(pd_4076)] {1to16} ; t15a pmulld m9, m16, [o(pd_401)] {1to16} ; t8a pmulld m18, m17, [o(pd_1189)] {1to16} ; t11a pmulld m17, [o(pd_3920)] {1to16} ; t12a psubd m18, m13, m18 REPX {paddd x, m13}, m22, m9, m17 REPX {psrad x, 12 }, m18, m22, m9, m17 mova m20, m9 mova m16, m18 mova m23, m22 mova m19, m17 jmp .main3 .main_fast_rect2: REPX {paddd x, m13}, m16, m17, m18, m19 REPX {psrad x, 12 }, m16, m17, m18, m19 .main_fast: pmulld m23, m16, [o(pd_4076)] {1to16} ; t15a pmulld m16, [o(pd_401)] {1to16} ; t8a pmulld m20, m19, [o(pd_2598)] {1to16} ; t9a pmulld m19, [o(pd_3166)] {1to16} ; t14a pmulld m22, m17, [o(pd_1189)] {1to16} ; t11a pmulld m17, [o(pd_3920)] {1to16} ; t12a pmulld m21, m18, [o(pd_3612)] {1to16} ; t13a pmulld m18, [o(pd_1931)] {1to16} ; t10a psubd m20, m13, m20 psubd m22, m13, m22 call .round2 jmp .main2 .main_rect2: call .round .main: ITX_MULSUB_2D 16, 23, 7, 9, 10, _, 401, 4076 ; t8a, t15a ITX_MULSUB_2D 20, 19, 7, 9, 10, _, 3166, 2598 ; t9a, t14a ITX_MULSUB_2D 22, 17, 7, 9, 10, _, 3920, 1189 ; t11a, t12a ITX_MULSUB_2D 18, 21, 7, 9, 10, _, 1931, 3612 ; t10a, t13a call .round .main2: paddd m9, m20, m16 ; t8 psubd m20, m16, m20 ; t9 psubd m16, m22, m18 ; t10 paddd m18, m22 ; t11 paddd m22, m23, m19 ; t15 psubd m23, m19 ; t14 psubd m19, m17, m21 ; t13 paddd m17, m21 ; t12 REPX {pmaxsd x, m14}, m20, m23, m16, m19 REPX {pminsd x, m15}, m20, m23, m16, m19 REPX {pmaxsd x, m14}, m9, m18, m22, m17 REPX {pminsd x, m15}, m9, m18, m22, m17 .main3: vpbroadcastd m11, [o(pd_3784)] vpbroadcastd m10, [o(pd_1567)] ITX_MULSUB_2D 23, 20, 21, 7, _, 13, 10, 11 ITX_MULSUB_2D 19, 16, 21, 7, _, 13, 10, 11, 2 paddd m21, m20, m19 ; t14 psubd m20, m19 ; t13 psubd m19, m9, m18 ; t11a paddd m9, m18 ; t8a psubd m18, m23, m16 ; t10 paddd m16, m23 ; t9 psubd m23, m22, m17 ; t12a paddd m22, m17 ; t15a REPX {pmaxsd x, m14}, m20, m23, m18, m19 REPX {pminsd x, m15}, m20, m23, m18, m19 REPX {pmulld x, m12}, m20, m23, m18, m19 psubd m7, m0, m6 ; dct8 out7 paddd m0, m6 ; dct8 out0 psubd m6, m1, m5 ; dct8 out6 paddd m1, m5 ; dct8 out1 REPX {pmaxsd x, m14}, m7, m0, m6, m1 psubd m5, m2, m4 ; dct8 out5 paddd m2, m4 ; dct8 out2 REPX {pminsd x, m15}, m7, m0, m6, m1 psubd m4, m3, m8 ; dct8 out4 paddd m3, m8 ; dct8 out3 REPX {pmaxsd x, m14}, m5, m2, m4, m3 paddd m20, m13 paddd m23, m13 REPX {pminsd x, m15}, m5, m2, m4, m3 psubd m17, m20, m18 ; t10a paddd m20, m18 ; t13a REPX {pmaxsd x, m14}, m22, m21, m16, m9 psubd m18, m23, m19 ; t11 paddd m19, m23 ; t12 REPX {pminsd x, m15}, m22, m21, m16, m9 REPX {psrad x, 12 }, m20, m19, m18, m17 ret .main_end: vpbroadcastd m11, [o(pd_2)] .main_end2: REPX {paddd x, m11}, m0, m1, m2, m3, m4, m5, m6, m7 psubd m23, m0, m22 ; out15 paddd m0, m22 ; out0 psubd m22, m1, m21 ; out14 paddd m1, m21 ; out1 psubd m21, m2, m20 ; out13 paddd m2, m20 ; out2 psubd m20, m3, m19 ; out12 paddd m3, m19 ; out3 psubd m19, m4, m18 ; out11 paddd m4, m18 ; out4 psubd m18, m5, m17 ; out10 paddd m5, m17 ; out5 psubd m17, m6, m16 ; out9 paddd m6, m16 ; out6 psubd m16, m7, m9 ; out8 paddd m7, m9 ; out7 REPX {vpsravd x, m11}, m0, m16, m1, m17, m2, m18, m3, m19, \ m4, m20, m5, m21, m6, m22, m7, m23 packssdw m0, m16 packssdw m1, m17 packssdw m2, m18 packssdw m3, m19 packssdw m4, m20 packssdw m5, m21 packssdw m6, m22 packssdw m7, m23 ret .main_end3: punpckhwd m8, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 punpcklwd m2, m3 punpckhwd m3, m4, m5 punpcklwd m4, m5 punpcklwd m5, m6, m7 punpckhwd m6, m7 punpckhdq m7, m0, m2 punpckldq m0, m2 punpckhdq m2, m8, m1 punpckldq m8, m1 punpckhdq m1, m4, m5 punpckldq m4, m5 punpckhdq m5, m3, m6 punpckldq m3, m6 vshufi32x4 m6, m0, m4, q3232 vinserti32x8 m0, ym4, 1 vinserti32x8 m4, m8, ym3, 1 vshufi32x4 m8, m3, q3232 vinserti32x8 m3, m7, ym1, 1 vshufi32x4 m7, m1, q3232 vshufi32x4 m1, m2, m5, q3232 vinserti32x8 m2, ym5, 1 vshufi32x4 m5, m7, m1, q2020 ; 10 11 vshufi32x4 m7, m1, q3131 ; 14 15 vshufi32x4 m1, m3, m2, q2020 ; 2 3 vshufi32x4 m3, m2, q3131 ; 6 7 vshufi32x4 m2, m0, m4, q3131 ; 4 5 vshufi32x4 m0, m4, q2020 ; 0 1 vshufi32x4 m4, m6, m8, q2020 ; 8 9 vshufi32x4 m6, m8, q3131 ; 12 13 ret ALIGN function_align .round: paddd m20, m13 paddd m22, m13 .round2: paddd m16, m13 paddd m18, m13 .round3: REPX {psrad x, 12 }, m16, m18, m20, m22 REPX {paddd x, m13}, m17, m19, m21, m23 REPX {psrad x, 12 }, m17, m19, m21, m23 ret INV_TXFM_16X16_FN adst, dct INV_TXFM_16X16_FN adst, flipadst INV_TXFM_16X16_FN adst, adst cglobal iadst_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp cmp eobd, 36 jl .fast call .main_pass1 packssdw m0, m16 packssdw m1, m17 packssdw m2, m18 packssdw m3, m19 packssdw m4, m5, m20 packssdw m5, m6, m21 packssdw m6, m7, m22 packssdw m7, m8, m23 jmp m(idct_16x16_internal_10bpc).pass1_end .fast: call .main_pass1_fast vpbroadcastd m9, [o(pd_2)] paddd m0, m9 psubd m1, m9, m1 paddd m2, m9 psubd m3, m9, m3 paddd m4, m9, m5 psubd m5, m9, m6 paddd m6, m9, m7 psubd m7, m9, m8 .pass1_fast_end: mova m9, [o(permA)] psrlq m8, m9, 8 REPX {psrad x, 2}, m0, m1, m2, m3, m4, m5, m6, m7 .pass1_fast_end2: mova m10, m9 mova m11, m8 call m(idct_16x8_internal_10bpc).transpose_16x8 pxor m4, m4 REPX {mova x, m4}, m5, m6, m7 REPX {mova [cq+64*x], ym4}, 0, 1, 2, 3, 4, 5, 6, 7 jmp tx2q .pass2: lea r5, [o_base_8bpc] call m(iadst_16x16_internal_8bpc).main_pass2b movshdup m12, [permC] mova m11, [pw_2048_m2048] psrlq m13, m12, 8 vpermq m8, m13, m0 vpermq m0, m12, m7 vpermq m7, m13, m1 vpermq m1, m12, m6 vpermq m6, m13, m2 vpermq m2, m12, m5 vpermq m5, m13, m3 vpermq m3, m12, m4 jmp m(idct_16x16_internal_10bpc).pass2_end ALIGN function_align .main_pass1: mova m0, [cq+64* 0] %if WIN64 movaps [cq+16*0], xmm6 movaps [cq+16*1], xmm7 %endif mova m23, [cq+64*15] vpbroadcastd m13, [o(pd_2048)] ITX_MULSUB_2D 23, 0, 8, 9, 10, 13, 201, 4091 ; t1 t0 mova m7, [cq+64* 7] mova m16, [cq+64* 8] ITX_MULSUB_2D 7, 16, 8, 9, 10, 13, 3035, 2751 ; t9 t8 mova m2, [cq+64* 2] mova m21, [cq+64*13] ITX_MULSUB_2D 21, 2, 8, 9, 10, 13, 995, 3973 ; t3 t2 mova m5, [cq+64* 5] mova m18, [cq+64*10] ITX_MULSUB_2D 5, 18, 8, 9, 10, 13, 3513, 2106 ; t11 t10 mova m4, [cq+64* 4] mova m19, [cq+64*11] ITX_MULSUB_2D 19, 4, 8, 9, 10, 13, 1751, 3703 ; t5 t4 mova m3, [cq+64* 3] mova m20, [cq+64*12] ITX_MULSUB_2D 3, 20, 8, 9, 10, 13, 3857, 1380 ; t13 t12 mova m6, [cq+64* 6] mova m17, [cq+64* 9] ITX_MULSUB_2D 17, 6, 8, 9, 10, 13, 2440, 3290 ; t7 t6 mova m1, [cq+64* 1] mova m22, [cq+64*14] ITX_MULSUB_2D 1, 22, 8, 9, 10, 13, 4052, 601 ; t15 t14 vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] psubd m9, m23, m7 ; t9a paddd m23, m7 ; t1a psubd m7, m2, m18 ; t10a paddd m18, m2 ; t2a REPX {pmaxsd x, m14}, m9, m23, m7, m18 psubd m2, m17, m1 ; t15a paddd m17, m1 ; t7a REPX {pminsd x, m15}, m9, m23, m7, m18 psubd m1, m21, m5 ; t11a paddd m21, m5 ; t3a REPX {pmaxsd x, m14}, m2, m17, m1, m21 psubd m5, m4, m20 ; t12a paddd m4, m20 ; t4a REPX {pminsd x, m15}, m2, m17, m1, m21 psubd m20, m19, m3 ; t13a paddd m19, m3 ; t5a REPX {pmaxsd x, m14}, m5, m4, m20, m19 psubd m8, m6, m22 ; t14a paddd m6, m22 ; t6a REPX {pminsd x, m15}, m5, m4, m20, m19 psubd m22, m0, m16 ; t8a paddd m16, m0 ; t0a REPX {pmaxsd x, m14}, m8, m6, m22, m16 vpbroadcastd m11, [o(pd_4017)] vpbroadcastd m10, [o(pd_799)] REPX {pminsd x, m15}, m8, m6, m22, m16 ITX_MULSUB_2D 22, 9, 0, 3, _, 13, 10, 11 ; t9 t8 ITX_MULSUB_2D 20, 5, 0, 3, _, 13, 11, 10 ; t12 t13 vpbroadcastd m11, [o(pd_2276)] vpbroadcastd m10, [o(pd_3406)] ITX_MULSUB_2D 7, 1, 0, 3, _, 13, 10, 11 ; t11 t10 ITX_MULSUB_2D 2, 8, 0, 3, _, 13, 11, 10 ; t14 t15 paddd m0, m16, m4 ; t0 psubd m16, m4 ; t4 psubd m3, m23, m19 ; t5 paddd m23, m19 ; t1 REPX {pmaxsd x, m14}, m0, m16, m3, m23 psubd m19, m18, m6 ; t6 paddd m18, m6 ; t2 REPX {pminsd x, m15}, m0, m16, m3, m23 psubd m6, m21, m17 ; t7 paddd m21, m17 ; t3 REPX {pmaxsd x, m14}, m19, m18, m6, m21 paddd m17, m9, m20 ; t8a psubd m9, m20 ; t12a REPX {pminsd x, m15}, m19, m18, m6, m21 psubd m20, m22, m5 ; t13a paddd m22, m5 ; t9a REPX {pmaxsd x, m14}, m17, m9, m20, m22 psubd m5, m1, m2 ; t14a paddd m1, m2 ; t10a REPX {pminsd x, m15}, m17, m9, m20, m22 psubd m2, m7, m8 ; t15a paddd m7, m8 ; t11a REPX {pmaxsd x, m14}, m5, m1, m2, m7 vpbroadcastd m11, [o(pd_3784)] vpbroadcastd m10, [o(pd_1567)] REPX {pminsd x, m15}, m5, m1, m2, m7 ITX_MULSUB_2D 16, 3, 4, 8, _, 13, 10, 11 ; t5a t4a ITX_MULSUB_2D 6, 19, 4, 8, _, 13, 11, 10 ; t6a t7a ITX_MULSUB_2D 9, 20, 4, 8, _, 13, 10, 11 ; t13 t12 ITX_MULSUB_2D 2, 5, 4, 8, _, 13, 11, 10 ; t14 t15 psubd m8, m0, m18 ; t2a paddd m0, m18 ; out0 psubd m18, m23, m21 ; t3a paddd m23, m21 ; -out15 paddd m21, m9, m5 ; -out13 psubd m9, m5 ; t15a psubd m5, m3, m6 ; t6 paddd m3, m6 ; -out3 REPX {pmaxsd x, m14}, m8, m18, m9, m5 psubd m6, m20, m2 ; t14a paddd m2, m20 ; out2 paddd m20, m16, m19 ; out12 psubd m16, m19 ; t7 REPX {pminsd x, m15}, m8, m18, m9, m5 psubd m19, m22, m7 ; t11 paddd m22, m7 ; out14 psubd m7, m17, m1 ; t10 paddd m1, m17 ; -out1 REPX {pmaxsd x, m14}, m6, m16, m19, m7 vpbroadcastd m12, [o(pd_1448)] vpbroadcastd m4, [o(pd_2)] vpbroadcastd m10, [o(pd_5120)] vpbroadcastd m11, [o(pd_5119)] REPX {pminsd x, m15}, m6, m16, m19, m7 psubd m17, m7, m19 ; -out9 paddd m7, m19 ; out6 psubd m19, m5, m16 ; -out11 paddd m5, m16 ; out4 REPX {pmulld x, m12}, m17, m7, m19, m5 psubd m16, m8, m18 ; out8 paddd m8, m18 ; -out7 psubd m18, m6, m9 ; out10 paddd m6, m9 ; -out5 REPX {pmulld x, m12}, m16, m8, m18, m6 REPX {paddd x, m4 }, m0, m2, m20, m22 REPX {psubd x, m4, x}, m1, m3, m21, m23 REPX {paddd x, m10 }, m7, m5, m16, m18 REPX {psubd x, m11, x}, m17, m19, m8, m6 REPX {psrad x, 2 }, m20, m22, m0, m2, m21, m23, m1, m3 REPX {psrad x, 13}, m17, m19, m5, m7, m16, m18, m6, m8 ret ALIGN function_align .main_pass1_fast: mova ym0, [cq+64*0] mova ym1, [cq+64*2] movshdup m8, [o(permB)] mova ym6, [cq+64*1] mova ym7, [cq+64*3] mova ym2, [cq+64*4] mova ym3, [cq+64*6] mova ym4, [cq+64*5] mova ym5, [cq+64*7] vpermt2q m0, m8, m1 ; 0 2 vpermt2q m7, m8, m6 ; 3 1 vpermt2q m2, m8, m3 ; 4 6 vpermt2q m5, m8, m4 ; 7 5 vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m12, [o(pd_2896)] jmp m(iadst_16x8_internal_10bpc).main_fast INV_TXFM_16X16_FN flipadst, dct INV_TXFM_16X16_FN flipadst, adst INV_TXFM_16X16_FN flipadst, flipadst cglobal iflipadst_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp cmp eobd, 36 jl .fast call m(iadst_16x16_internal_10bpc).main_pass1 packssdw m4, m19, m3 packssdw m3, m20, m5 packssdw m5, m18, m2 packssdw m2, m21, m6 packssdw m6, m17, m1 packssdw m1, m22, m7 packssdw m7, m16, m0 packssdw m0, m23, m8 jmp m(idct_16x16_internal_10bpc).pass1_end .fast: call m(iadst_16x16_internal_10bpc).main_pass1_fast vpbroadcastd m9, [o(pd_2)] psubd m4, m9, m3 paddd m3, m9, m5 paddd m5, m9, m2 psubd m2, m9, m6 psubd m6, m9, m1 paddd m1, m9, m7 paddd m7, m9, m0 psubd m0, m9, m8 jmp m(iadst_16x16_internal_10bpc).pass1_fast_end .pass2: lea r5, [o_base_8bpc] call m(iadst_16x16_internal_8bpc).main_pass2b movshdup m12, [permC] movu m11, [pw_m2048_2048] psrlq m13, m12, 8 vpermq m8, m13, m7 vpermq m7, m13, m6 vpermq m6, m13, m5 vpermq m5, m13, m4 vpermq m3, m12, m3 vpermq m2, m12, m2 vpermq m1, m12, m1 vpermq m0, m12, m0 jmp m(idct_16x16_internal_10bpc).pass2_end INV_TXFM_16X16_FN identity, dct, -92 INV_TXFM_16X16_FN identity, identity cglobal iidentity_16x16_internal_10bpc, 0, 7, 16, dst, stride, c, eob, tx2 %undef cmp vpbroadcastd m10, [o(pd_5793)] vpbroadcastd m11, [o(pd_5120)] mov r6, cq cmp eobd, 36 jl .fast call .pass1_main packssdw m0, m6, m8 packssdw m1, m7, m9 call .pass1_main packssdw m2, m6, m8 packssdw m3, m7, m9 call .pass1_main packssdw m4, m6, m8 packssdw m5, m7, m9 call .pass1_main packssdw m6, m8 packssdw m7, m9 jmp m(idct_16x16_internal_10bpc).pass1_end2 .fast: call .pass1_main_fast packssdw m0, m6, m7 call .pass1_main_fast packssdw m1, m6, m7 call .pass1_main_fast packssdw m2, m6, m7 call .pass1_main_fast packssdw m3, m6, m7 punpckhwd m4, m0, m1 punpcklwd m0, m1 punpckhwd m1, m2, m3 punpcklwd m2, m3 punpckldq m3, m4, m1 punpckhdq m4, m1 punpckhdq m1, m0, m2 punpckldq m0, m2 pxor m7, m7 vshufi32x4 m2, m0, m3, q3131 vshufi32x4 m0, m3, q2020 vshufi32x4 m3, m1, m4, q3131 vshufi32x4 m1, m4, q2020 REPX {mova x, m7}, m4, m5, m6 jmp m(idct_16x16_internal_10bpc).pass1_end3 .pass2: movshdup m14, [o(permC)] vpbroadcastd m15, [o(pw_1697x16)] lea r6, [strideq*3] vpbroadcastd m11, [o(pw_2048)] pxor m12, m12 vpbroadcastd m13, [pixel_10bpc_max] vpermq m8, m14, m0 vpermq m9, m14, m1 call .pass2_main vpermq m8, m14, m2 vpermq m9, m14, m3 call .pass2_main vpermq m8, m14, m4 vpermq m9, m14, m5 call .pass2_main vpermq m8, m14, m6 vpermq m9, m14, m7 .pass2_main: pmulhrsw m0, m15, m8 pmulhrsw m1, m15, m9 paddsw m8, m8 paddsw m9, m9 paddsw m8, m0 paddsw m9, m1 jmp m(idct_16x8_internal_10bpc).write_16x4 ALIGN function_align .pass1_main: pmulld m6, m10, [r6+64*0] pmulld m7, m10, [r6+64*1] pmulld m8, m10, [r6+64*8] pmulld m9, m10, [r6+64*9] add r6, 64*2 REPX {paddd x, m11}, m6, m7, m8, m9 REPX {psrad x, 13 }, m6, m8, m7, m9 ret ALIGN function_align .pass1_main_fast: mova ym6, [r6+64* 0] vinserti32x8 m6, [r6+64* 4], 1 mova ym7, [r6+64* 8] vinserti32x8 m7, [r6+64*12], 1 add r6, 64 REPX {pmulld x, m10}, m6, m7 REPX {paddd x, m11}, m6, m7 REPX {psrad x, 13 }, m6, m7 ret cglobal inv_txfm_add_dct_dct_8x32_10bpc, 4, 7, 22, dst, stride, c, eob %undef cmp lea r5, [o_base] test eobd, eobd jz .dconly vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] vpbroadcastd m11, [o(pd_2)] mova m20, [o(idct8x32p)] pxor m21, m21 cmp eobd, 43 jl .fast call .pass1_main punpcklwd m16, m0, m1 punpcklwd m17, m2, m3 punpckhwd m18, m0, m1 punpckhwd m19, m2, m3 cmp eobd, 107 jge .full punpckldq m0, m16, m17 ; 0 2 punpckhdq m1, m16, m17 ; 4 6 punpckldq m2, m18, m19 ; 8 10 punpckhdq m3, m18, m19 ; 12 14 lea r5, [o_base_8bpc] vextracti32x8 ym14, m0, 1 vextracti32x8 ym15, m1, 1 vextracti32x8 ym16, m2, 1 vextracti32x8 ym17, m3, 1 call m(idct_8x16_internal_8bpc).main_fast call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast jmp .end .full: add cq, 64 call .pass1_main punpcklwd m5, m0, m1 punpcklwd m6, m2, m3 punpckhwd m7, m0, m1 punpckhwd m8, m2, m3 punpckldq m0, m16, m17 ; 0 2 punpckhdq m1, m16, m17 ; 4 6 punpckldq m2, m18, m19 ; 8 10 punpckhdq m3, m18, m19 ; 12 14 punpckldq m4, m5, m6 ; 16 18 punpckhdq m5, m6 ; 20 22 punpckldq m6, m7, m8 ; 24 26 punpckhdq m7, m8 ; 28 30 lea r5, [o_base_8bpc] vextracti32x8 ym14, m0, 1 vextracti32x8 ym15, m1, 1 vextracti32x8 ym16, m2, 1 vextracti32x8 ym17, m3, 1 vextracti32x8 ym18, m4, 1 vextracti32x8 ym19, m5, 1 vextracti32x8 ym20, m6, 1 vextracti32x8 ym21, m7, 1 call m(idct_8x16_internal_8bpc).main REPX {pshufd x, x, q1032}, ym18, ym19, ym20, ym21 call m(inv_txfm_add_dct_dct_8x32_8bpc).main jmp .end .fast: movshdup m8, [o(permB)] mova ym1, [cq+128*1] mova ym5, [cq+128*5] mova ym7, [cq+128*3] mova ym3, [cq+128*7] mova ym0, [cq+128*0] mova ym4, [cq+128*2] mova ym2, [cq+128*4] mova ym6, [cq+128*6] vpermt2q m1, m8, m5 ; 1 5 vpermt2q m3, m8, m7 ; 7 3 vpermt2q m0, m8, m4 ; 0 2 vpermt2q m2, m8, m6 ; 4 6 mova [cq+128*0], ym21 REPX {vmovdqa32 [cq+128*x], ym21}, 1, 2, 3, 4, 5, 6, 7 call m(idct_8x8_internal_10bpc).main call m(idct_8x8_internal_10bpc).main_end packssdw m0, m2 packssdw m1, m3 vpermb m0, m20, m0 vprold m20, 16 vpermb m2, m20, m1 punpckhdq m1, m0, m2 punpckldq m0, m2 lea r5, [o_base_8bpc] vextracti32x8 ym14, m0, 1 vextracti32x8 ym15, m1, 1 call m(idct_8x16_internal_8bpc).main_fast2 call m(inv_txfm_add_dct_dct_8x32_8bpc).main_fast2 .end: call m(inv_txfm_add_dct_dct_8x32_8bpc).main_end ; performs vzeroupper lea r3, [strideq*2] vpbroadcastd m12, [pixel_10bpc_max] lea r6, [strideq*3] pxor m11, m11 lea r3, [dstq+r3*8] pmulhrsw m0, m10 pmulhrsw m1, m10 call .write_8x4x2 pmulhrsw m0, m10, m2 pmulhrsw m1, m10, m3 call .write_8x4x2 pmulhrsw m0, m10, m4 pmulhrsw m1, m10, m5 call .write_8x4x2 pmulhrsw m0, m10, m6 pmulhrsw m1, m10, m7 .write_8x4x2: mova xm8, [dstq+strideq*0] vinserti32x4 ym8, [dstq+strideq*1], 1 vinserti32x4 m8, [dstq+strideq*2], 2 vinserti32x4 m8, [dstq+r6 ], 3 mova xm9, [r3 +r6 ] vinserti32x4 ym9, [r3 +strideq*2], 1 vinserti32x4 m9, [r3 +strideq*1], 2 vinserti32x4 m9, [r3 +strideq*0], 3 paddw m8, m0 paddw m9, m1 pmaxsw m8, m11 pmaxsw m9, m11 pminsw m8, m12 pminsw m9, m12 mova [dstq+strideq*0], xm8 vextracti32x4 [dstq+strideq*1], ym8, 1 vextracti32x4 [dstq+strideq*2], m8, 2 vextracti32x4 [dstq+r6 ], m8, 3 lea dstq, [dstq+strideq*4] vextracti32x4 [r3 +strideq*0], m9, 3 vextracti32x4 [r3 +strideq*1], m9, 2 vextracti32x4 [r3 +strideq*2], ym9, 1 mova [r3 +r6 ], xm9 lea r3, [r3+strideq*4] ret .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 32 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_8x8_10bpc).dconly2 ALIGN function_align .pass1_main: mova m0, [cq+128*0] mova m1, [cq+128*1] mova m2, [cq+128*2] mova m3, [cq+128*3] mova m4, [cq+128*4] mova m5, [cq+128*5] mova m6, [cq+128*6] mova m7, [cq+128*7] REPX {mova [cq+128*x], m21}, 0, 1, 2, 3, 4, 5, 6, 7 call m(idct_8x16_internal_10bpc).main call m(idct_8x16_internal_10bpc).main_end2 packssdw m0, m4 packssdw m1, m5 packssdw m2, m6 packssdw m3, m7 REPX {vpermb x, m20, x}, m0, m1, m2, m3 ret cglobal inv_txfm_add_identity_identity_8x32_10bpc, 4, 8, 12, dst, stride, c, eob vpbroadcastd m9, [pw_5] lea r4, [strideq*3] pxor m10, m10 lea r5, [strideq*5] vpbroadcastd m11, [pixel_10bpc_max] sub eobd, 107 lea r6, [strideq+r4*2] .loop: mova m0, [cq+128*0] packssdw m0, [cq+128*1] mova m1, [cq+128*2] packssdw m1, [cq+128*3] mova m2, [cq+128*4] packssdw m2, [cq+128*5] mova m3, [cq+128*6] packssdw m3, [cq+128*7] lea r7, [dstq+strideq*8] REPX {mova [cq+128*x], m10}, 0, 1, 2, 3 REPX {paddsw x, m9}, m0, m1, m2, m3 REPX {mova [cq+128*x], m10}, 4, 5, 6, 7 REPX {psraw x, 3 }, m0, m1, m2, m3 add cq, 64 mova xm4, [dstq+strideq*0] mova xm5, [dstq+strideq*1] mova xm6, [dstq+strideq*2] mova xm7, [dstq+r4 *1] punpckhwd m8, m0, m1 vinserti32x4 ym4, [dstq+strideq*4], 1 punpcklwd m0, m1 vinserti32x4 ym5, [dstq+r5 *1], 1 punpckhwd m1, m2, m3 vinserti32x4 ym6, [dstq+r4 *2], 1 punpcklwd m2, m3 vinserti32x4 ym7, [dstq+r6 *1], 1 punpckhwd m3, m0, m8 vinserti32x4 m4, [r7 +strideq*0], 2 punpcklwd m0, m8 vinserti32x4 m5, [r7 +strideq*1], 2 punpckhwd m8, m2, m1 vinserti32x4 m6, [r7 +strideq*2], 2 punpcklwd m2, m1 vinserti32x4 m7, [r7 +r4 *1], 2 punpckhqdq m1, m0, m2 vinserti32x4 m4, [r7 +strideq*4], 3 punpcklqdq m0, m2 vinserti32x4 m5, [r7 +r5 *1], 3 punpcklqdq m2, m3, m8 vinserti32x4 m6, [r7 +r4 *2], 3 punpckhqdq m3, m8 vinserti32x4 m7, [r7 +r6 *1], 3 paddw m0, m4 paddw m1, m5 paddw m2, m6 paddw m3, m7 REPX {pmaxsw x, m10}, m0, m1, m2, m3 REPX {pminsw x, m11}, m0, m1, m2, m3 mova [dstq+strideq*0], xm0 mova [dstq+strideq*1], xm1 mova [dstq+strideq*2], xm2 mova [dstq+r4 *1], xm3 vextracti32x4 [dstq+strideq*4], ym0, 1 vextracti32x4 [dstq+r5 *1], ym1, 1 vextracti32x4 [dstq+r4 *2], ym2, 1 vextracti32x4 [dstq+r6 *1], ym3, 1 lea dstq, [r7+strideq*8] vextracti32x4 [r7 +strideq*0], m0, 2 vextracti32x4 [r7 +strideq*1], m1, 2 vextracti32x4 [r7 +strideq*2], m2, 2 vextracti32x4 [r7 +r4 *1], m3, 2 vextracti32x4 [r7 +strideq*4], m0, 3 vextracti32x4 [r7 +r5 *1], m1, 3 vextracti32x4 [r7 +r4 *2], m2, 3 vextracti32x4 [r7 +r6 *1], m3, 3 add eobd, 0x80000000 jnc .loop RET cglobal inv_txfm_add_dct_dct_32x8_10bpc, 4, 7, 0, dst, stride, c, eob %undef cmp lea r5, [o_base] test eobd, eobd jz .dconly mova m11, [o(permB)] mova m0, [cq+64* 0] ; 0 1 mova m4, [cq+64* 1] ; 2 3 mova m1, [cq+64* 2] ; 4 5 mova m8, [cq+64* 3] ; 6 7 vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] psrlq m10, m11, 32 %if WIN64 movaps [cq+16*0], xmm6 movaps [cq+16*1], xmm7 %endif mova m16, m11 vpermi2q m16, m0, m1 ; 1 5 mova m17, m11 vpermi2q m17, m8, m4 ; 7 3 cmp eobd, 43 jl .fast mova m18, [cq+64* 4] ; 8 9 mova m20, [cq+64* 5] ; 10 11 mova m6, [cq+64* 6] ; 12 13 mova m7, [cq+64* 7] ; 14 15 vpermt2q m0, m10, m18 ; 0 8 vpermt2q m18, m11, m6 ; 9 13 mova m19, m11 vpermi2q m19, m7, m20 ; 15 11 cmp eobd, 107 jge .full vpermt2q m1, m10, m6 ; 4 12 vpermt2q m4, m10, m8 ; 2 6 vpermt2q m7, m10, m20 ; 14 10 mov r6d, 64*1 call m(idct_8x8_internal_10bpc).main_fast call m(idct_16x8_internal_10bpc).main_fast call .main_fast call m(idct_16x16_internal_10bpc).main_end jmp .end .full: mova m2, [cq+64* 8] ; 16 17 mova m5, [cq+64* 9] ; 18 19 mova m9, [cq+64*10] ; 20 21 mova m21, [cq+64*11] ; 22 23 vpermt2q m1, m10, m9 ; 4 20 vpermt2q m7, m10, m21 ; 14 22 vpermt2q m21, m11, m5 ; 23 19 vpermt2q m5, m10, m20 ; 18 10 mova m20, m11 vpermi2q m20, m2, m9 ; 17 21 mova m22, [cq+64*12] ; 24 25 mova m9, [cq+64*13] ; 26 27 mova m3, [cq+64*14] ; 28 29 mova m23, [cq+64*15] ; 30 31 vpermt2q m2, m10, m22 ; 16 24 vpermt2q m22, m11, m3 ; 25 29 vpermt2q m3, m10, m6 ; 28 12 vpermt2q m4, m10, m9 ; 2 26 mova m6, m10 vpermi2q m6, m23, m8 ; 30 6 vpermt2q m23, m11, m9 ; 31 27 mov r6d, 64*3 call m(idct_8x8_internal_10bpc).main call m(idct_16x8_internal_10bpc).main call .main call m(idct_16x16_internal_10bpc).main_end jmp .end .fast: vpermq m0, m10, m0 ; 0 0 vpermq m1, m10, m1 ; 4 4 vpermt2q m4, m10, m8 ; 2 6 xor r6d, r6d call .main_fast2 call m(idct_16x16_internal_10bpc).main_end .end: %if WIN64 movaps xmm6, [cq+16*0] movaps xmm7, [cq+16*1] %endif vzeroupper call .transpose_8x32 pxor m14, m14 .zero_loop: mova [cq+r6*4+64*3], m14 mova [cq+r6*4+64*2], m14 mova [cq+r6*4+64*1], m14 mova [cq+r6*4+64*0], m14 sub r6d, 64 jge .zero_loop lea r5, [o_base_8bpc] punpckhqdq m1, m0, m2 punpcklqdq m0, m2 punpcklqdq m2, m3, m4 punpckhqdq m3, m4 punpcklqdq m4, m5, m7 punpckhqdq m5, m7 punpckhqdq m7, m6, m8 punpcklqdq m6, m8 call m(inv_txfm_add_dct_dct_32x8_8bpc).main pxor m12, m12 .write_32x8_start: vpbroadcastd m11, [pw_2048] vpbroadcastd m13, [pixel_10bpc_max] lea r3, [strideq*3] .write_32x8: pmulhrsw m0, m11 pmulhrsw m1, m11 pmulhrsw m2, m11 pmulhrsw m3, m11 call .write_32x4 pmulhrsw m0, m11, m4 pmulhrsw m1, m11, m5 pmulhrsw m2, m11, m6 pmulhrsw m3, m11, m7 .write_32x4: paddw m0, [dstq+strideq*0] paddw m1, [dstq+strideq*1] paddw m2, [dstq+strideq*2] paddw m3, [dstq+r3 ] REPX {pmaxsw x, m12}, m0, m1, m2, m3 REPX {pminsw x, m13}, m0, m1, m2, m3 mova [dstq+strideq*0], m0 mova [dstq+strideq*1], m1 mova [dstq+strideq*2], m2 mova [dstq+r3 ], m3 lea dstq, [dstq+strideq*4] ret .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 8 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_32x16_10bpc).dconly2 ALIGN function_align .main_fast3: ; assuming m0=in0 in0, m4=in2 in2, and m16=in1 in3 vbroadcasti32x4 m5, [o(pd_401_4076)] pmulld m3, m0, m12 pmulld m4, m5 REPX {paddd x, m13}, m3, m4 REPX {psrad x, 12 }, m3, m4 ; m3=idct8:t0-7, m4=t8a t15a ; t8a t15a -> t8/9 t14/15 vbroadcasti32x4 m5, [o(pd_3784_m3784)] pshufd m7, m4, q1032 pmulld m6, m4, [o(pd_1567)]{bcstd} pmulld m5, m7 paddd m6, m13 paddd m5, m6 psrad m5, 12 ; m5=t9a t14a ; t14a t9a -> t13/14 t9/10 [m5] & t8 15 -> t8/11a t12/15a [m4] shufps m6, m4, m5, q1032 ; t12 t13 shufps m8, m4, m5, q3210 ; t11a t10 pmulld m9, m6, m12 pmulld m7, m8, m12 paddd m9, m13 paddd m5, m9, m7 ; t12 t13a psubd m4, m9, m7 ; t11 t10a REPX {psrad x, 12 }, m5, m4 psubd m7, m3, m6 ; dct16 out15 out14 paddd m0, m3, m6 ; dct16 out0 out1 psubd m6, m3, m5 ; dct16 out12 out13 paddd m1, m3, m5 ; dct16 out3 out2 psubd m5, m3, m4 ; dct16 out11 out10 paddd m2, m3, m4 ; dct16 out4 out5 psubd m4, m3, m8 ; dct16 out8 out9 paddd m3, m8 ; dct16 out7 out6 REPX {pmaxsd x, m14}, m7, m0, m6, m1, m5, m2, m4, m3 REPX {pminsd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7 ; idct32_bottomhalf vbroadcasti32x4 m18, [o(pd_201_m601)] vbroadcasti32x4 m19, [o(pd_4091_4052)] pmulld m17, m16, m19 pmulld m16, m18 REPX {paddd x, m13}, m17, m16 REPX {psrad x, 12 }, m17, m16 ; m17: t31a t24a -> t30/31 t24/25, m16: t16a t23a -> t16/17 t22/23 [step2] vbroadcasti32x4 m10, [o(pd_799_m2276)] vbroadcasti32x4 m11, [o(pd_4017_3406)] pmulld m18, m17, m10 pmulld m19, m17, m11 pmulld m8, m16, m11 pmulld m9, m16, m10 REPX {paddd x, m13}, m18, m19 psubd m18, m8 paddd m19, m9 REPX {psrad x, 12 }, m18, m19 ; m17=t31 t24 -> t28/31a t24/27a, m16=t16 t23 -> t16/19a t20/23a ; m18=t17a t22a -> t17/18 t21/22, m19=t30a t25a -> t29/30 t25/26 punpckhqdq m23, m17, m19 ; t24a t25 [or t27a t26] punpcklqdq m20, m16, m18 ; t16a t17 [or t19a t18] punpckhqdq m22, m16, m18 ; t23a t22 [or t20a t21] punpcklqdq m16, m17, m19 ; t28a t29 [or t31a t30] mova m21, m23 mova m18, m20 mova m17, m22 mova m19, m16 jmp .main4 .main_fast2: ; bottom three-quarters are zero vbroadcasti32x4 m8, [o(pd_799_4017)] pmulld m8, m1 ; t4 t7 vpmulld m0, [o(pd_2896)] {1to16} ; t0 t1 REPX {paddd x, m13}, m8, m0 REPX {psrad x, 12 }, m8, m0 pmulld m3, m8, m12 mova m2, m0 ; t3 t2 call m(idct_8x8_internal_10bpc).main3 vbroadcasti32x4 m6, [o(pd_4076_3920)] vbroadcasti32x4 m3, [o(pd_401_m1189)] pmulld m6, m4 ; t15 t12 pmulld m4, m3 ; t9 t10 REPX {paddd x, m13}, m6, m4 REPX {psrad x, 12 }, m6, m4 mova m5, m6 ; t14 t13 mova m9, m4 ; t8 t11 call m(idct_16x8_internal_10bpc).main3 vbroadcasti32x4 m23, [o(pd_4091_3973)] vbroadcasti32x4 m7, [o(pd_201_995)] vbroadcasti32x4 m22, [o(pd_1380_601)] vbroadcasti32x4 m9, [o(pd_3857_4052)] pmulld m23, m16 ; t16 t20 pmulld m16, m7 ; t31 t27 pmulld m22, m17 ; -t19 -t25 pmulld m17, m9 ; t28 t24 REPX {paddd x, m13}, m23, m16, m17 psubd m22, m13, m22 REPX {psrad x, 12 }, m23, m16, m22, m17 mova m20, m23 ; t30 t26 mova m9, m16 ; t17 t21 mova m19, m22 ; t18 t22 mova m18, m17 ; t29 t25 jmp .main3 .main_fast: ; bottom half is zero vbroadcasti32x4 m23, [o(pd_4091_3973)] vbroadcasti32x4 m7, [o(pd_201_995)] vbroadcasti32x4 m20, [o(pd_2751_2106)] vbroadcasti32x4 m9, [o(pd_3035_3513)] vbroadcasti32x4 m21, [o(pd_3703_3290)] vbroadcasti32x4 m10, [o(pd_1751_2440)] vbroadcasti32x4 m22, [o(pd_1380_601)] vbroadcasti32x4 m11, [o(pd_3857_4052)] pmulld m23, m16 ; t16a t20a pmulld m16, m7 ; t31a t27a pmulld m20, m19 ; -t17a -t21a pmulld m19, m9 ; t30a t26a pmulld m21, m18 ; t18a t22a pmulld m18, m10 ; t29a t25a pmulld m22, m17 ; -t19a -t25a pmulld m17, m11 ; t28a t24a psubd m20, m13, m20 psubd m22, m13, m22 jmp .main2 .main: ITX_MULSUB_2D 16, 23, 7, 9, 10, _, 201_995, 4091_3973 ITX_MULSUB_2D 20, 19, 7, 9, 10, _, 3035_3513, 2751_2106 ITX_MULSUB_2D 18, 21, 7, 9, 10, _, 1751_2440, 3703_3290 ITX_MULSUB_2D 22, 17, 7, 9, 10, _, 3857_4052, 1380_601 paddd m20, m13 paddd m22, m13 .main2: REPX {paddd x, m13}, m16, m23, m19 REPX {psrad x, 12 }, m16, m20, m23, m19 psubd m9, m16, m20 ; t17 t21 paddd m16, m20 ; t16 t20 psubd m20, m23, m19 ; t30 t26 paddd m23, m19 ; t31 t27 REPX {pmaxsd x, m14}, m9, m16, m20, m23 REPX {paddd x, m13}, m21, m18, m17 REPX {psrad x, 12 }, m18, m22, m21, m17 psubd m19, m22, m18 ; t18 t22 paddd m22, m18 ; t19 t23 psubd m18, m17, m21 ; t29 t25 paddd m17, m21 ; t28 t24 REPX {pmaxsd x, m14}, m19, m22, m18, m17 REPX {pminsd x, m15}, m20, m9, m18, m19, m16, m23, m22, m17 .main3: vbroadcasti32x4 m11, [o(pd_4017_2276)] vbroadcasti32x4 m10, [o(pd_799_3406)] psubd m7, m0, m6 ; dct16 out15 out14 paddd m0, m6 ; dct16 out0 out1 psubd m6, m1, m5 ; dct16 out12 out13 paddd m1, m5 ; dct16 out3 out2 psubd m5, m2, m4 ; dct16 out11 out10 paddd m2, m4 ; dct16 out4 out5 psubd m4, m3, m8 ; dct16 out8 out9 paddd m3, m8 ; dct16 out7 out6 ITX_MULSUB_2D 20, 9, 8, 21, _, 13, 10, 11 ITX_MULSUB_2D 18, 19, 8, 21, _, 13, 10, 11, 2 REPX {pmaxsd x, m14}, m7, m0, m6, m1, m5, m2, m4, m3 punpckhqdq m21, m16, m20 ; t20 t21a punpcklqdq m16, m20 ; t16 t17a punpcklqdq m20, m22, m19 ; t19 t18a punpckhqdq m22, m19 ; t23 t22a REPX {pminsd x, m15}, m0, m1, m2, m3, m4, m5, m6, m7 punpcklqdq m19, m23, m9 ; t31 t30a punpckhqdq m23, m9 ; t27 t26a punpckhqdq m9, m17, m18 ; t24 t25a punpcklqdq m17, m18 ; t28 t29a psubd m18, m16, m20 ; t19a t18 paddd m20, m16 ; t16a t17 psubd m16, m19, m17 ; t28a t29 paddd m19, m17 ; t31a t30 psubd m17, m22, m21 ; t20a t21 paddd m22, m21 ; t23a t22 psubd m21, m9, m23 ; t27a t26 paddd m23, m9 ; t24a t25 REPX {pmaxsd x, m14}, m18, m16, m17, m21 REPX {pminsd x, m15}, m16, m18, m21, m17 REPX {pmaxsd x, m14}, m20, m22, m19, m23 REPX {pminsd x, m15}, m20, m22, m19, m23 .main4: vpbroadcastd m11, [o(pd_3784)] vpbroadcastd m10, [o(pd_1567)] ITX_MULSUB_2D 16, 18, 8, 9, _, 13, 10, 11 ITX_MULSUB_2D 21, 17, 8, 9, _, 13, 10, 11, 2 paddd m9, m20, m22 ; t16 t17a psubd m20, m22 ; t23 t22a paddd m22, m19, m23 ; t31 t30a psubd m19, m23 ; t24 t25a psubd m23, m16, m17 ; t20a t21 paddd m16, m17 ; t19a t18 psubd m17, m18, m21 ; t27a t26 paddd m21, m18 ; t28a t29 REPX {pmaxsd x, m14}, m20, m19, m23, m17 REPX {pminsd x, m15}, m19, m20, m17, m23 REPX {pmulld x, m12}, m19, m20, m17, m23 REPX {pmaxsd x, m14}, m22, m21, m16, m9 paddd m19, m13 paddd m17, m13 REPX {pminsd x, m15}, m22, m21, m16, m9 psubd m18, m19, m20 ; t23a t22 paddd m19, m20 ; t24a t25 paddd m20, m17, m23 ; t27 t26a psubd m17, m23 ; t20 t21a REPX {psrad x, 12 }, m20, m19, m18, m17 ret .transpose_8x32: mova m10, [o(idct32x8p)] psrlw m8, m10, 8 mova m9, m8 vpermi2w m8, m1, m5 vpermt2w m1, m10, m5 vprold m5, m9, 16 vpermi2w m9, m3, m7 vpermt2w m3, m10, m7 vprold m10, 16 mova m7, m5 vpermi2w m5, m0, m4 vpermt2w m0, m10, m4 vpermi2w m7, m2, m6 vpermt2w m2, m10, m6 punpckhdq m6, m5, m8 punpckldq m5, m8 punpckhdq m8, m7, m9 punpckldq m7, m9 punpckhdq m4, m2, m3 punpckldq m2, m3 punpckhdq m3, m0, m1 punpckldq m0, m1 ret cglobal inv_txfm_add_identity_identity_32x8_10bpc, 4, 7, 10, dst, stride, c, eob vpbroadcastd m5, [pw_4096] lea r4, [strideq*3] mova m6, [idtx32x8p] lea r5, [strideq*5] vpbroadcastd m9, [pixel_10bpc_max] lea r6, [strideq+r4*2] pxor m8, m8 sub eobd, 107 psrlw m7, m6, 8 .loop: mova m0, [cq+64*0] packssdw m0, [cq+64*1] ; 02 13 mova m1, [cq+64*2] packssdw m1, [cq+64*3] ; 46 57 mova m2, [cq+64*4] packssdw m2, [cq+64*5] ; 8a 9b mova m3, [cq+64*6] packssdw m3, [cq+64*7] ; ce df REPX {pmulhrsw x, m5}, m0, m1, m2, m3 REPX {mova [cq+64*x], m8}, 0, 1, 2, 3 mova m4, m6 vpermi2w m4, m1, m3 vpermt2w m1, m7, m3 REPX {mova [cq+64*x], m8}, 4, 5, 6, 7 mova m3, m7 vpermi2w m3, m0, m2 vpermt2w m0, m6, m2 add cq, 64*8 punpcklqdq m2, m3, m1 ; 4 5 punpckhqdq m3, m1 ; 6 7 punpckhqdq m1, m0, m4 ; 2 3 punpcklqdq m0, m4 ; 0 1 mova ym4, [dstq+strideq*0] vinserti32x8 m4, [dstq+strideq*1], 1 paddw m0, m4 mova ym4, [dstq+strideq*2] vinserti32x8 m4, [dstq+r4 *1], 1 paddw m1, m4 mova ym4, [dstq+strideq*4] vinserti32x8 m4, [dstq+r5 *1], 1 paddw m2, m4 mova ym4, [dstq+r4 *2] vinserti32x8 m4, [dstq+r6 *1], 1 paddw m3, m4 REPX {pmaxsw x, m8}, m0, m1, m2, m3 REPX {pminsw x, m9}, m0, m1, m2, m3 mova [dstq+strideq*0], ym0 vextracti32x8 [dstq+strideq*1], m0, 1 mova [dstq+strideq*2], ym1 vextracti32x8 [dstq+r4 *1], m1, 1 mova [dstq+strideq*4], ym2 vextracti32x8 [dstq+r5 *1], m2, 1 mova [dstq+r4 *2], ym3 vextracti32x8 [dstq+r6 *1], m3, 1 add dstq, 32 add eobd, 0x80000000 jnc .loop RET cglobal inv_txfm_add_dct_dct_16x32_10bpc, 4, 7, 0, dst, stride, c, eob %undef cmp lea r5, [o_base] test eobd, eobd jz .dconly vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] %if WIN64 movaps [rsp+ 8], xmm6 movaps [rsp+24], xmm7 %endif cmp eobd, 36 jl .fast call .pass1 cmp eobd, 151 jge .full lea r5, [o_base_8bpc] pxor m9, m9 punpcklwd m8, m1, m1 ; 2 punpckhwd m14, m1, m1 ; 3 punpcklwd m1, m3, m3 ; 6 punpckhwd m15, m3, m3 ; 7 punpcklwd m3, m6, m6 ; 12 punpckhwd m19, m6, m6 ; 13 punpcklwd m6, m9, m4 ; __ 8 punpckhwd m20, m4, m4 ; 9 punpckhwd m16, m5, m5 ; 11 punpcklwd m5, m5 ; 10 punpcklwd m9, m0 ; __ 0 punpckhwd m21, m0, m0 ; 1 punpcklwd m0, m7, m7 ; 14 punpckhwd m17, m7, m7 ; 15 punpcklwd m7, m2, m2 ; 4 punpckhwd m18, m2, m2 ; 5 call m(idct_16x16_internal_8bpc).main_fast call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast mov r6d, 64*3 pxor m8, m8 .zero_loop: REPX {mova [cq+r6*8+128*x], m8}, 3, 2, 1, 0 sub r6d, 64 jge .zero_loop jmp .pass2_end .full: mova [cq+128*0], m0 mova [cq+128*1], m1 mova [cq+128*2], m2 mova [cq+128*3], m3 mova [cq+128*4], m4 mova [cq+128*5], m5 mova [cq+128*6], m6 mova [cq+128*7], m7 add cq, 64 call .pass1 mova m9, [cq-64* 1] ; 0 1 mova m14, [cq+64* 1] ; 2 3 mova m18, [cq+64* 3] ; 4 5 mova m15, [cq+64* 5] ; 6 7 mova m20, [cq+64* 7] ; 8 9 mova m16, [cq+64* 9] ; 10 11 mova m22, [cq+64*11] ; 12 13 mova m19, [cq+64*13] ; 14 15 lea r5, [o_base_8bpc] punpcklwd m8, m7, m14 ; 30 2 punpckhwd m21, m7, m9 ; 31 1 punpcklwd m7, m6, m18 ; 28 4 punpckhwd m14, m6 ; 3 29 punpcklwd m9, m0, m9 ; 16 0 punpckhwd m17, m19, m0 ; 15 17 punpcklwd m0, m19, m1 ; 14 18 punpckhwd m19, m1, m22 ; 19 13 punpcklwd m1, m15, m5 ; 6 26 punpckhwd m18, m5, m18 ; 27 5 punpcklwd m6, m4, m20 ; 24 8 punpckhwd m15, m4 ; 7 25 punpcklwd m5, m3, m16 ; 22 10 punpckhwd m20, m3, m20 ; 23 9 punpcklwd m3, m22, m2 ; 12 20 punpckhwd m16, m2 ; 11 21 call m(idct_16x16_internal_8bpc).main2 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf mov r6d, 32*7 pxor m8, m8 .full_zero_loop: REPX {mova [cq+r6*8+64*x], m8}, 2, 1, 0, -1 sub r6d, 32 jge .full_zero_loop jmp .pass2_end .fast: mova ym0, [cq+128*0] mova ym2, [cq+128*4] movshdup m8, [o(permB)] mova ym1, [cq+128*2] mova ym3, [cq+128*6] mova ym4, [cq+128*1] mova ym5, [cq+128*3] mova ym6, [cq+128*5] mova ym7, [cq+128*7] vpermt2q m0, m8, m2 ; 0 4 vpermt2q m1, m8, m3 ; 2 6 vpermt2q m4, m8, m5 ; 1 3 vpermt2q m7, m8, m6 ; 7 5 REPX {pmulld x, m12}, m0, m1, m4, m7 pxor ym16, ym16 mova [cq+128*0], ym16 REPX {vmovdqa32 [cq+128*x], ym16}, 1, 2, 3, 4, 5, 6, 7 REPX {paddd x, m13}, m0, m1, m4, m7 REPX {psrad x, 12 }, m0, m1, m4, m7 call m(idct_8x8_internal_10bpc).main_fast call m(idct_16x8_internal_10bpc).main_fast vpbroadcastd m11, [o(pd_1)] call m(idct_8x16_internal_10bpc).main_end2 mova m8, [o(idct8x32p)] packssdw m0, m4 packssdw m1, m5 packssdw m2, m6 packssdw m3, m7 mova m6, [dup16_perm] vpermb m0, m8, m0 vpermb m2, m8, m2 vprold m8, 16 vpermb m1, m8, m1 vpermb m3, m8, m3 punpckldq m4, m0, m2 punpckhdq m0, m2 punpckldq m2, m1, m3 punpckhdq m1, m3 punpckldq m21, m4, m2 punpckhdq m14, m4, m2 punpckldq m18, m0, m1 punpckhdq m15, m0, m1 vpermb m8, m6, m14 ; 2 vpermb m1, m6, m15 ; 6 vpermb m7, m6, m18 ; 4 pmovzxwd m9, ym21 ; 0 vpord m6, [o(pb_32)] {1to16} lea r5, [o_base_8bpc] vpermb m21, m6, m21 ; 1 vpermb m15, m6, m15 ; 7 vpermb m18, m6, m18 ; 5 vpermb m14, m6, m14 ; 3 pslld m9, 16 call m(idct_16x16_internal_8bpc).main_fast2 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 .pass2_end: movshdup m22, [permC] vpbroadcastd m11, [pw_2048] vpbroadcastd m13, [pixel_10bpc_max] lea r6, [strideq*3] pxor m12, m12 psrlq m23, m22, 8 vpermq m8, m22, m0 vpermq m9, m23, m1 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m22, m2 vpermq m9, m23, m3 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m22, m4 vpermq m9, m23, m5 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m22, m6 vpermq m9, m23, m7 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m22, m14 vpermq m9, m23, m15 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m22, m16 vpermq m9, m23, m17 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m22, m18 vpermq m9, m23, m19 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m22, m20 vpermq m9, m23, m21 %if WIN64 movaps xmm6, [rsp+ 8] movaps xmm7, [rsp+24] %endif vzeroupper jmp m(idct_16x8_internal_10bpc).write_16x4 .pass1: pmulld m0, m12, [cq+128* 0] pmulld m1, m12, [cq+128* 2] pmulld m2, m12, [cq+128* 4] pmulld m3, m12, [cq+128* 6] pmulld m4, m12, [cq+128* 8] pmulld m5, m12, [cq+128*10] pmulld m6, m12, [cq+128*12] pmulld m7, m12, [cq+128*14] call m(idct_8x16_internal_10bpc).main_rect2 pmulld m16, m12, [cq+128* 1] pmulld m17, m12, [cq+128* 3] pmulld m18, m12, [cq+128* 5] pmulld m19, m12, [cq+128* 7] pmulld m20, m12, [cq+128* 9] pmulld m21, m12, [cq+128*11] pmulld m22, m12, [cq+128*13] pmulld m23, m12, [cq+128*15] call m(idct_16x16_internal_10bpc).main_rect2 vpbroadcastd m11, [o(pd_1)] call m(idct_16x16_internal_10bpc).main_end2 jmp m(idct_16x16_internal_10bpc).main_end3 .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 32 jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly cglobal inv_txfm_add_identity_identity_16x32_10bpc, 4, 7, 16, dst, stride, c, eob %undef cmp vpbroadcastd m10, [pw_2896x8] vpbroadcastd m11, [pw_1697x16] vpbroadcastd m13, [pw_8192] vpbroadcastd m15, [pixel_10bpc_max] lea r6, [strideq*9] pxor m14, m14 paddw m12, m13, m13 ; pw_16384 cmp eobd, 151 jl .main call .main add cq, 64-128*4 lea dstq, [dstq+strideq*8] .main: call .main_internal add cq, 128*4 pmulhrsw m1, m13, m2 pmulhrsw m3, m13, m4 pmulhrsw m5, m13, m6 pmulhrsw m7, m13, m8 call .main_internal .main2: pmulhrsw m2, m13 pmulhrsw m4, m13 pmulhrsw m6, m13 pmulhrsw m8, m13 punpcklqdq m0, m1, m2 ; 0 8 punpckhqdq m1, m2 ; 1 9 call .write_16x2x2 punpcklqdq m0, m3, m4 ; 2 10 punpckhqdq m1, m3, m4 ; 3 11 call .write_16x2x2 punpcklqdq m0, m5, m6 ; 4 12 punpckhqdq m1, m5, m6 ; 5 13 call .write_16x2x2 punpcklqdq m0, m7, m8 ; 6 14 punpckhqdq m1, m7, m8 ; 7 15 .write_16x2x2: mova ym2, [dstq+strideq*0] vinserti32x8 m2, [dstq+strideq*8], 1 mova ym9, [dstq+strideq*1] vinserti32x8 m9, [dstq+r6 ], 1 paddw m0, m2 paddw m1, m9 pmaxsw m0, m14 pmaxsw m1, m14 pminsw m0, m15 pminsw m1, m15 mova [dstq+strideq*0], ym0 vextracti32x8 [dstq+strideq*8], m0, 1 mova [dstq+strideq*1], ym1 vextracti32x8 [dstq+r6 ], m1, 1 lea dstq, [dstq+strideq*2] ret .main_internal: mova m8, [cq+128* 0] packssdw m8, [cq+128* 8] mova m6, [cq+128* 1] packssdw m6, [cq+128* 9] mova m0, [cq+128* 2] packssdw m0, [cq+128*10] mova m2, [cq+128* 3] packssdw m2, [cq+128*11] REPX {pmulhrsw x, m10}, m8, m6, m0, m2 REPX {vpermq x, x, q3120}, m8, m6, m0, m2 pmulhrsw m4, m11, m8 pmulhrsw m9, m11, m6 REPX {mova [cq+128*x], m14}, 0, 1, 2, 3 pmulhrsw m4, m12 pmulhrsw m9, m12 paddsw m8, m4 paddsw m6, m9 pmulhrsw m4, m11, m0 pmulhrsw m9, m11, m2 REPX {mova [cq+128*x], m14}, 8, 9, 10, 11 pmulhrsw m4, m12 pmulhrsw m9, m12 paddsw m0, m4 paddsw m2, m9 punpcklwd m4, m8, m6 punpckhwd m8, m6 punpcklwd m6, m0, m2 punpckhwd m0, m2 punpckldq m2, m4, m6 ; 0 1 punpckhdq m4, m6 ; 2 3 punpckldq m6, m8, m0 ; 4 5 punpckhdq m8, m0 ; 6 7 ret cglobal inv_txfm_add_dct_dct_32x16_10bpc, 4, 7, 0, dst, stride, c, eob %undef cmp lea r5, [o_base] test eobd, eobd jz .dconly vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] %if WIN64 movaps [rsp+ 8], xmm6 movaps [rsp+24], xmm7 %endif mov r6d, 8*12 cmp eobd, 36 jl .fast pmulld m0, m12, [cq+64* 0] pmulld m1, m12, [cq+64* 4] pmulld m2, m12, [cq+64* 8] pmulld m3, m12, [cq+64*12] pmulld m16, m12, [cq+64* 2] pmulld m17, m12, [cq+64* 6] pmulld m18, m12, [cq+64*10] pmulld m19, m12, [cq+64*14] cmp eobd, 151 jge .full call m(idct_8x16_internal_10bpc).main_fast_rect2 call m(idct_16x16_internal_10bpc).main_fast_rect2 call .idct16_sumsub call .pass1_load_spill call .main_fast_rect2 jmp .pass1_end .full: pmulld m4, m12, [cq+64*16] pmulld m5, m12, [cq+64*20] pmulld m6, m12, [cq+64*24] pmulld m7, m12, [cq+64*28] pmulld m20, m12, [cq+64*18] pmulld m21, m12, [cq+64*22] pmulld m22, m12, [cq+64*26] pmulld m23, m12, [cq+64*30] add r6d, 8*16 call m(idct_8x16_internal_10bpc).main_rect2 call m(idct_16x16_internal_10bpc).main_rect2 call .idct16_sumsub call .pass1_load_spill pmulld m16, m12, [cq+64*17] pmulld m17, m12, [cq+64*19] pmulld m18, m12, [cq+64*21] pmulld m19, m12, [cq+64*23] pmulld m20, m12, [cq+64*25] pmulld m21, m12, [cq+64*27] pmulld m22, m12, [cq+64*29] pmulld m23, m12, [cq+64*31] call .main_rect2 .pass1_end: vpbroadcastd m11, [o(pd_1)] lea r4, [cq+64] call .idct32_pass1_end lea r5, [o_base_8bpc] punpckhqdq m19, m5, m16 ; 11 punpcklqdq m5, m16 ; 10 punpckhqdq m16, m2, m1 ; 5 punpcklqdq m2, m1 ; 4 punpcklqdq m1, m15, m4 ; 2 punpckhqdq m15, m4 ; 3 punpcklqdq m4, m14, m18 ; 8 punpckhqdq m18, m14, m18 ; 9 punpckhqdq m14, m0, m20 ; 1 punpcklqdq m0, m20 ; 0 punpckhqdq m20, m6, m17 ; 13 punpcklqdq m6, m17 ; 12 punpckhqdq m17, m3, m21 ; 7 punpcklqdq m3, m21 ; 6 punpckhqdq m21, m7, m8 ; 15 punpcklqdq m7, m8 ; 14 call m(inv_txfm_add_dct_dct_32x8_8bpc).main call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf jmp .end .fast: pmulld ym0, ym12, [cq+64*0] pmulld ym1, ym12, [cq+64*4] movshdup m7, [o(permB)] mova ym4, [cq+64*2] mova ym5, [cq+64*6] mova ym16, [cq+64*1] mova ym2, [cq+64*5] mova ym3, [cq+64*3] mova ym17, [cq+64*7] vpermt2q m4, m7, m5 ; 2 6 vpermt2q m16, m7, m2 ; 1 5 vpermt2q m17, m7, m3 ; 7 3 paddd ym0, ym13 paddd ym1, ym13 psrad ym0, 12 psrad ym1, 12 vpermq m0, m7, m0 ; 0 0 vpermq m1, m7, m1 ; 4 4 REPX {pmulld x, m12}, m4, m16, m17 REPX {paddd x, m13}, m4, m16, m17 REPX {psrad x, 12 }, m4, m16, m17 call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast2 vpbroadcastd m11, [o(pd_1)] call m(idct_16x16_internal_10bpc).main_end2 call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 lea r5, [o_base_8bpc] punpckhqdq m14, m0, m2 ; 1 punpcklqdq m0, m2 ; 0 punpcklqdq m1, m3, m4 ; 2 punpckhqdq m15, m3, m4 ; 3 punpcklqdq m2, m5, m7 ; 4 punpckhqdq m16, m5, m7 ; 5 punpcklqdq m3, m6, m8 ; 6 punpckhqdq m17, m6, m8 ; 7 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast .end: %if WIN64 movaps xmm6, [rsp+ 8] movaps xmm7, [rsp+24] %endif pxor m12, m12 .zero_loop: mova [cq+r6*8+64*3], m12 mova [cq+r6*8+64*2], m12 mova [cq+r6*8+64*1], m12 mova [cq+r6*8+64*0], m12 sub r6d, 8*4 jge .zero_loop call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8_start pmulhrsw m0, m11, m14 pmulhrsw m1, m11, m15 pmulhrsw m2, m11, m16 pmulhrsw m3, m11, m17 call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 pmulhrsw m0, m11, m18 pmulhrsw m1, m11, m19 pmulhrsw m2, m11, m20 pmulhrsw m3, m11, m21 vzeroupper jmp m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 16 .dconly3: add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 384 sar r6d, 9 .dconly2: vpbroadcastd m3, [o(dconly_10bpc)] imul r6d, 181 add r6d, 2176 sar r6d, 12 vpbroadcastw m2, r6d paddsw m2, m3 .dconly_loop: paddsw m0, m2, [dstq+strideq*0] paddsw m1, m2, [dstq+strideq*1] psubusw m0, m3 psubusw m1, m3 mova [dstq+strideq*0], m0 mova [dstq+strideq*1], m1 lea dstq, [dstq+strideq*2] sub r3d, 2 jg .dconly_loop RET ALIGN function_align .idct16_sumsub: psubd m23, m0, m22 ; t15 paddd m0, m22 ; t0 psubd m22, m1, m21 ; t14 paddd m1, m21 ; t1 REPX {pmaxsd x, m14}, m23, m0, m22, m1 psubd m21, m2, m20 ; t13 paddd m2, m20 ; t2 REPX {pminsd x, m15}, m23, m0, m22, m1 psubd m20, m3, m19 ; t12 paddd m3, m19 ; t3 REPX {pmaxsd x, m14}, m21, m2, m20, m3 psubd m19, m4, m18 ; t11 paddd m4, m18 ; t4 REPX {pminsd x, m15}, m21, m2, m20, m3 psubd m18, m5, m17 ; t10 paddd m5, m17 ; t5 REPX {pmaxsd x, m14}, m19, m4, m18, m5 psubd m17, m6, m16 ; t9 paddd m6, m16 ; t6 REPX {pminsd x, m15}, m19, m4, m18, m5 psubd m16, m7, m9 ; t8 paddd m7, m9 ; t7 REPX {pmaxsd x, m14}, m17, m6, m16, m7 REPX {pminsd x, m15}, m17, m6, m16, m7 ret .idct32_pass1_end: psrlq m12, [o(permC)], 24 ; 0 2 8 10 1 3 9 11 psrlq m13, m12, 32 ; 4 6 12 14 5 7 13 15 %macro IDCT32_PASS1_END 2 ; low, high paddd m8, m11, [r4+128*%1] paddd m9, m11, [cq+128*%1] psubd m10, m8, m%1 ; out 16+n paddd m8, m%1 ; out 15-n paddd m%1, m9, m%2 ; out 0+n psubd m9, m%2 ; out 31-n REPX {vpsravd x, m11}, m10, m%1, m8, m9 packssdw m%1, m10 ; 0+n 16+n packssdw m%2, m8, m9 ; 15-n 31-n %endmacro IDCT32_PASS1_END 0, 23 ; 0 16, 15 31 IDCT32_PASS1_END 7, 16 ; 7 23, 8 24 IDCT32_PASS1_END 1, 22 ; 1 17, 14 30 IDCT32_PASS1_END 6, 17 ; 6 22, 9 25 IDCT32_PASS1_END 2, 21 ; 2 18, 13 29 IDCT32_PASS1_END 5, 18 ; 5 21, 10 26 IDCT32_PASS1_END 3, 20 ; 3 19, 12 28 IDCT32_PASS1_END 4, 19 ; 4 20, 11 27 .transpose_16x32: mova m14, m13 vpermi2q m14, m0, m16 vpermt2q m0, m12, m16 mova m15, m13 vpermi2q m15, m1, m17 vpermt2q m1, m12, m17 mova m16, m13 vpermi2q m16, m2, m18 vpermt2q m2, m12, m18 mova m17, m13 vpermi2q m17, m3, m19 vpermt2q m3, m12, m19 mova m18, m13 vpermi2q m18, m4, m20 vpermt2q m4, m12, m20 mova m19, m13 vpermi2q m19, m5, m21 vpermt2q m5, m12, m21 mova m20, m13 vpermi2q m20, m6, m22 vpermt2q m6, m12, m22 mova m21, m13 vpermi2q m21, m7, m23 vpermt2q m7, m12, m23 punpckhwd m8, m2, m3 ; c04 d04 c05 d05 c06 d06 c07 d07 punpcklwd m2, m3 ; c00 d00 c01 d01 c02 d02 c03 d03 punpckhwd m3, m0, m1 ; a04 b04 a05 b05 a06 b06 a07 b07 punpcklwd m0, m1 ; a00 b00 a01 b01 a02 b02 a03 b03 punpckhwd m1, m4, m5 ; e04 f04 e05 f05 e06 f06 e07 f07 punpcklwd m4, m5 ; e00 f00 e01 f01 e02 f02 e03 f03 punpckhwd m5, m6, m7 ; g04 h04 g05 h05 g06 h06 g07 h07 punpcklwd m6, m7 ; g00 h00 g01 h01 g02 h02 g03 h03 punpckhwd m7, m14, m15 ; a12 b12 a13 b13 a14 b14 a15 b15 punpcklwd m14, m15 ; a08 b08 a09 b09 a10 b10 a11 b11 punpckhwd m15, m16, m17 ; c12 d12 c13 d13 c14 d14 c15 d15 punpcklwd m16, m17 ; c08 d08 c09 d09 c10 d10 c11 d11 punpckhwd m17, m18, m19 ; e12 f12 e13 f13 e14 f14 e15 f15 punpcklwd m18, m19 ; e08 f08 e09 f09 e10 f10 e11 f11 punpckhwd m19, m20, m21 ; g12 h12 g13 h13 g14 h14 g15 h15 punpcklwd m20, m21 ; g08 h08 g09 h09 g10 h10 g11 h11 punpckhdq m21, m1, m5 ; e06 f06 g06 h06 e07 f07 g07 h07 punpckldq m1, m5 ; e04 f04 g04 h04 e05 f05 g05 h05 punpckhdq m5, m14, m16 ; a10 b10 c10 d10 a11 b11 c11 d11 punpckldq m14, m16 ; a08 b08 c08 d08 a09 b09 c09 d09 punpckhdq m16, m18, m20 ; e10 f10 g10 h10 e11 f11 g11 h11 punpckldq m18, m20 ; e08 f08 g08 h08 e09 f09 g09 h09 punpckldq m20, m4, m6 ; e00 f00 g00 h00 e01 f01 g01 h01 punpckhdq m4, m6 ; e02 f02 g02 h02 e03 f03 g03 h03 punpckldq m6, m7, m15 ; a12 b12 c12 d12 a13 b13 c13 d13 punpckhdq m7, m15 ; a14 b14 c14 d14 a15 b15 c15 d15 punpckhdq m15, m0, m2 ; a02 b02 c02 d02 a03 b03 c03 d03 punpckldq m0, m2 ; a00 b00 c00 d00 a01 b01 c01 d01 punpckldq m2, m3, m8 ; a04 b04 c04 d04 a05 b05 c05 d05 punpckhdq m3, m8 ; a06 b06 c06 d06 a07 b07 c07 d07 punpckhdq m8, m17, m19 ; e14 f14 g14 h14 e15 f15 g15 h15 punpckldq m17, m19 ; e12 f12 g12 h12 e13 f13 g13 h13 ret .pass1_load_spill: mova [cq+64* 0], m0 mova [cq+64* 2], m1 mova [cq+64* 4], m2 mova [cq+64* 6], m3 mova [cq+64* 8], m4 mova [cq+64*10], m5 mova [cq+64*12], m6 mova [cq+64*14], m7 pmulld m0, m12, [cq+64* 1] pmulld m1, m12, [cq+64* 3] pmulld m2, m12, [cq+64* 5] pmulld m3, m12, [cq+64* 7] pmulld m4, m12, [cq+64* 9] pmulld m5, m12, [cq+64*11] pmulld m6, m12, [cq+64*13] pmulld m7, m12, [cq+64*15] mova [cq+64* 1], m23 mova [cq+64* 3], m22 mova [cq+64* 5], m21 mova [cq+64* 7], m20 mova [cq+64* 9], m19 mova [cq+64*11], m18 mova [cq+64*13], m17 mova [cq+64*15], m16 ret .main_fast2_rect2: REPX {paddd x, m13}, m0, m1, m2, m3 REPX {psrad x, 12 }, m0, m1, m2, m3 .main_fast2: ; bottom 3/4 is zero pmulld m23, m0, [o(pd_4091)] {1to16} ; t31a pmulld m0, [o(pd_201)] {1to16} ; t16a pmulld m20, m3, [o(pd_1380)] {1to16} ; t19a pmulld m3, [o(pd_3857)] {1to16} ; t28a pmulld m21, m2, [o(pd_3973)] {1to16} ; t27a pmulld m2, [o(pd_995)] {1to16} ; t20a pmulld m6, m1, [o(pd_601)] {1to16} ; t23a pmulld m17, m1, [o(pd_4052)] {1to16} ; t24a REPX {psubd x, m13, x}, m20, m6 REPX {paddd x, m13}, m23, m0, m3, m21, m2, m17 REPX {psrad x, 12 }, m20, m6, m23, m0, m3, m21, m2, m17 mova m8, m0 mova m16, m23 mova m7, m20 mova m4, m3 mova m19, m2 mova m18, m21 mova m5, m6 mova m22, m17 jmp .main3 .main_fast_rect2: call m(idct_8x16_internal_10bpc).round .main_fast: ; bottom half is zero pmulld m23, m0, [o(pd_4091)] {1to16} ; t31a pmulld m0, [o(pd_201)] {1to16} ; t16a pmulld m16, m7, [o(pd_2751)] {1to16} ; t17a pmulld m7, [o(pd_3035)] {1to16} ; t30a pmulld m19, m4, [o(pd_3703)] {1to16} ; t29a pmulld m4, [o(pd_1751)] {1to16} ; t18a pmulld m20, m3, [o(pd_1380)] {1to16} ; t19a pmulld m3, [o(pd_3857)] {1to16} ; t28a pmulld m21, m2, [o(pd_3973)] {1to16} ; t27a pmulld m2, [o(pd_995)] {1to16} ; t20a pmulld m18, m5, [o(pd_2106)] {1to16} ; t21a pmulld m5, [o(pd_3513)] {1to16} ; t26a pmulld m17, m6, [o(pd_3290)] {1to16} ; t25a pmulld m6, [o(pd_2440)] {1to16} ; t22a pmulld m22, m1, [o(pd_601)] {1to16} ; t23a pmulld m1, [o(pd_4052)] {1to16} ; t24a REPX {psubd x, m13, x}, m16, m20, m18, m22 call m(idct_16x16_internal_10bpc).round3 jmp .main2 .main_rect2: call m(idct_8x16_internal_10bpc).round call m(idct_16x16_internal_10bpc).round .main: ITX_MULSUB_2D 0, 23, 8, 9, 10, _, 201, 4091 ; t16a, t31a ITX_MULSUB_2D 16, 7, 8, 9, 10, _, 3035, 2751 ; t17a, t30a ITX_MULSUB_2D 4, 19, 8, 9, 10, _, 1751, 3703 ; t18a, t29a ITX_MULSUB_2D 20, 3, 8, 9, 10, _, 3857, 1380 ; t19a, t28a ITX_MULSUB_2D 2, 21, 8, 9, 10, _, 995, 3973 ; t20a, t27a ITX_MULSUB_2D 18, 5, 8, 9, 10, _, 3513, 2106 ; t21a, t26a ITX_MULSUB_2D 6, 17, 8, 9, 10, _, 2440, 3290 ; t22a, t25a ITX_MULSUB_2D 22, 1, 8, 9, 10, _, 4052, 601 ; t23a, t24a call m(idct_16x16_internal_10bpc).round .main2: call m(idct_8x16_internal_10bpc).round psubd m8, m0, m16 ; t17 paddd m0, m16 ; t16 psubd m16, m23, m7 ; t30 paddd m23, m7 ; t31 REPX {pmaxsd x, m14}, m8, m0, m16, m23 paddd m7, m20, m4 ; t19 psubd m20, m4 ; t18 REPX {pminsd x, m15}, m8, m0, m16, m23 paddd m4, m3, m19 ; t28 psubd m3, m19 ; t29 REPX {pmaxsd x, m14}, m7, m20, m4, m3 psubd m19, m2, m18 ; t21 paddd m2, m18 ; t20 REPX {pminsd x, m15}, m7, m20, m4, m3 psubd m18, m21, m5 ; t26 paddd m21, m5 ; t27 REPX {pmaxsd x, m14}, m19, m2, m18, m21 psubd m5, m22, m6 ; t22 paddd m6, m22 ; t23 REPX {pminsd x, m15}, m19, m2, m18, m21 psubd m22, m1, m17 ; t25 paddd m17, m1 ; t24 REPX {pmaxsd x, m14}, m5, m6, m22, m17 REPX {pminsd x, m15}, m5, m6, m22, m17 .main3: vpbroadcastd m11, [o(pd_4017)] vpbroadcastd m10, [o(pd_799)] ITX_MULSUB_2D 16, 8, 9, 1, _, 13, 10, 11 ; t17a, t30a ITX_MULSUB_2D 3, 20, 9, 1, _, 13, 10, 11, 2 ; t29a, t18a vpbroadcastd m11, [o(pd_2276)] vpbroadcastd m10, [o(pd_3406)] ITX_MULSUB_2D 18, 19, 9, 1, _, 13, 10, 11 ; t21a, t26a ITX_MULSUB_2D 22, 5, 9, 1, _, 13, 10, 11, 2 ; t25a, t22a paddd m1, m6, m2 ; t23a psubd m6, m2 ; t20a psubd m2, m17, m21 ; t27a paddd m17, m21 ; t24a REPX {pmaxsd x, m14}, m1, m6, m2, m17 psubd m21, m23, m4 ; t28a paddd m23, m4 ; t31a REPX {pminsd x, m15}, m1, m6, m2, m17 psubd m4, m16, m20 ; t18 paddd m16, m20 ; t17 REPX {pmaxsd x, m14}, m21, m23, m4, m16 psubd m20, m0, m7 ; t19a paddd m0, m7 ; t16a REPX {pminsd x, m15}, m21, m23, m4, m16 psubd m7, m8, m3 ; t29 paddd m3, m8 ; t30 REPX {pmaxsd x, m14}, m20, m0, m7, m3 paddd m8, m5, m18 ; t22 psubd m5, m18 ; t21 REPX {pminsd x, m15}, m20, m0, m7, m3 psubd m18, m22, m19 ; t26 paddd m22, m19 ; t25 REPX {pmaxsd x, m14}, m8, m5, m18, m22 vpbroadcastd m11, [o(pd_3784)] vpbroadcastd m10, [o(pd_1567)] REPX {pminsd x, m15}, m8, m5, m18, m22 ITX_MULSUB_2D 21, 20, 9, 19, _, 13, 10, 11 ; t19, t28 ITX_MULSUB_2D 2, 6, 9, 19, _, 13, 10, 11, 2 ; t27, t20 ITX_MULSUB_2D 7, 4, 9, 19, _, 13, 10, 11 ; t18a, t29a ITX_MULSUB_2D 18, 5, 9, 19, _, 13, 10, 11, 2 ; t26a, t21a psubd m19, m0, m1 ; t23 paddd m0, m1 ; t16 paddd m1, m8, m16 ; t17a psubd m8, m16, m8 ; t22a REPX {pmaxsd x, m14}, m19, m0, m1, m8 psubd m16, m23, m17 ; t24 paddd m23, m17 ; t31 REPX {pminsd x, m15}, m19, m0, m1, m8 psubd m17, m3, m22 ; t25a paddd m22, m3 ; t30a REPX {pmaxsd x, m14}, m16, m23, m17, m22 paddd m3, m6, m21 ; t19a psubd m6, m21, m6 ; t20a REPX {pminsd x, m15}, m16, m23, m17, m22 paddd m21, m18, m4 ; t29 psubd m18, m4, m18 ; t26 REPX {pmaxsd x, m14}, m3, m6, m21, m18 psubd m4, m20, m2 ; t27a paddd m20, m2 ; t28a REPX {pminsd x, m15}, m3, m6, m21, m18 paddd m2, m7, m5 ; t18 psubd m7, m5 ; t21 REPX {pmaxsd x, m14}, m4, m20, m2, m7 REPX {pminsd x, m15}, m4, m20, m2, m7 REPX {pmulld x, m12}, m18, m16, m4, m17, m7, m19, m6, m8 REPX {paddd x, m13}, m18, m16, m4, m17 psubd m5, m18, m7 ; t21a paddd m18, m7 ; t26a psubd m7, m16, m19 ; t23a paddd m16, m19 ; t24a REPX {psrad x, 12 }, m5, m18, m7, m16 paddd m19, m4, m6 ; t27 psubd m4, m6 ; t20 psubd m6, m17, m8 ; t22 paddd m17, m8 ; t25 REPX {psrad x, 12 }, m19, m4, m6, m17 ret cglobal inv_txfm_add_identity_identity_32x16_10bpc, 4, 7, 16, dst, stride, c, eob %undef cmp vpbroadcastd m10, [pw_2896x8] vpbroadcastd m11, [pw_1697x16] vpbroadcastd m13, [pw_2048] vpbroadcastd m15, [pixel_10bpc_max] lea r6, [strideq*9] pxor m14, m14 cmp eobd, 151 jl .main mov r4, dstq call .main add cq, 64*12 lea dstq, [r4+32] .main: call .main_internal add cq, 64*4 pmulhrsw m1, m13, m2 pmulhrsw m3, m13, m4 pmulhrsw m5, m13, m6 pmulhrsw m7, m13, m8 call .main_internal jmp m(inv_txfm_add_identity_identity_16x32_10bpc).main2 .main_internal: mova m8, [cq+64* 0] packssdw m8, [cq+64* 8] mova m6, [cq+64* 1] packssdw m6, [cq+64* 9] mova m0, [cq+64* 2] packssdw m0, [cq+64*10] mova m2, [cq+64* 3] packssdw m2, [cq+64*11] REPX {pmulhrsw x, m10}, m8, m6, m0, m2 REPX {paddsw x, x }, m8, m6, m0, m2 REPX {vpermq x, x, q3120}, m8, m6, m0, m2 pmulhrsw m4, m11, m8 pmulhrsw m9, m11, m6 paddsw m8, m8 paddsw m6, m6 REPX {mova [cq+64*x], m14}, 0, 1, 2, 3 paddsw m8, m4 paddsw m6, m9 pmulhrsw m4, m11, m0 pmulhrsw m9, m11, m2 paddsw m0, m0 paddsw m2, m2 REPX {mova [cq+64*x], m14}, 8, 9, 10, 11 paddsw m0, m4 paddsw m2, m9 punpcklwd m4, m8, m6 punpckhwd m8, m6 punpcklwd m6, m0, m2 punpckhwd m0, m2 punpckldq m2, m4, m6 ; 0 1 punpckhdq m4, m6 ; 2 3 punpckldq m6, m8, m0 ; 4 5 punpckhdq m8, m0 ; 6 7 ret cglobal inv_txfm_add_dct_dct_32x32_10bpc, 4, 7, 0, dst, stride, c, eob %undef cmp lea r5, [o_base] test eobd, eobd jz .dconly vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] WIN64_SPILL_XMM 30 cmp eobd, 136 jl .fast add cq, 64 cmp eobd, 543 jge .full call .pass1_fast ; bottomright 16x16 zero mov r6d, 16*12 jmp .lefthalf .full: call .pass1 mov r6d, 16*28 .lefthalf: mova [cq+128* 0], m0 mova [cq+128* 1], m1 mova [cq+128* 2], m2 mova [cq+128* 3], m3 mova [cq+128* 4], m14 mova [cq+128* 5], m15 mova [cq+128* 6], m16 mova [cq+128* 7], m17 mova [cq+128* 8], m22 mova [cq+128* 9], m23 mova [cq+128*10], m24 mova [cq+128*11], m25 mova [cq+128*12], m26 mova [cq+128*13], m27 mova [cq+128*14], m28 mova [cq+128*15], m29 sub cq, 64 vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] call .pass1 lea r5, [o_base_8bpc] call .pass2_start pxor m12, m12 .right_zero_loop: mova [cq+r6*8+64+128*3], m12 mova [cq+r6*8+64+128*2], m12 mova [cq+r6*8+64+128*1], m12 mova [cq+r6*8+64+128*0], m12 sub r6d, 16*4 jge .right_zero_loop mov r6d, 16*28 jmp .end2 .pass2_start: mova m4, [cq+64+128* 0] mova m5, [cq+64+128* 1] mova m6, [cq+64+128* 2] mova m7, [cq+64+128* 3] mova m18, [cq+64+128* 4] mova m19, [cq+64+128* 5] mova m20, [cq+64+128* 6] mova m21, [cq+64+128* 7] call m(inv_txfm_add_dct_dct_32x8_8bpc).main call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf mova [cq+128*0], m14 mova [cq+128*1], m15 mova [cq+128*2], m16 mova [cq+128*3], m17 mova [cq+128*4], m18 mova [cq+128*5], m19 mova [cq+128*6], m20 mova [cq+128*7], m21 mova m14, [cq+64+128* 8] mova m15, [cq+64+128* 9] mova m16, [cq+64+128*10] mova m17, [cq+64+128*11] mova m18, [cq+64+128*12] mova m19, [cq+64+128*13] mova m20, [cq+64+128*14] mova m21, [cq+64+128*15] jmp m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf .fast: ; topleft 16x16 nonzero cmp eobd, 36 jl .fast2 call .pass1_fast lea r5, [o_base_8bpc] call .pass2_fast_start jmp .end .pass2_fast_start: call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast mova [cq+128*0], m14 mova [cq+128*1], m15 mova [cq+128*2], m16 mova [cq+128*3], m17 mova [cq+128*4], m18 mova [cq+128*5], m19 mova [cq+128*6], m20 mova [cq+128*7], m21 jmp m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast .fast2: ; topleft 8x8 nonzero movshdup m7, [o(permB)] mova ym0, [cq+128*0] mova ym1, [cq+128*4] mova ym4, [cq+128*2] mova ym5, [cq+128*6] mova ym16, [cq+128*1] mova ym2, [cq+128*5] mova ym3, [cq+128*3] mova ym17, [cq+128*7] mov r6d, 16*4 vpermq m0, m7, m0 ; 0 0 vpermq m1, m7, m1 ; 4 4 vpermt2q m4, m7, m5 ; 2 6 vpermt2q m16, m7, m2 ; 1 5 vpermt2q m17, m7, m3 ; 7 3 call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast2 call m(idct_16x16_internal_10bpc).main_end call .pass2_fast2_start .end: pxor m12, m12 .end2: call .pass2_end .zero_loop: mova [cq+r6*8+128*3], m12 mova [cq+r6*8+128*2], m12 mova [cq+r6*8+128*1], m12 mova [cq+r6*8+128*0], m12 sub r6d, 16*4 jge .zero_loop WIN64_RESTORE_XMM vzeroupper ret .pass2_fast2_start: call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 lea r5, [o_base_8bpc] punpckhqdq m22, m0, m2 ; 1 punpcklqdq m0, m2 ; 0 punpcklqdq m1, m5, m7 ; 4 punpckhqdq m24, m5, m7 ; 5 punpcklqdq m14, m3, m4 ; 2 punpckhqdq m23, m3, m4 ; 3 punpcklqdq m15, m6, m8 ; 6 punpckhqdq m25, m6, m8 ; 7 mova m10, m13 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2 mova [cq+128*0], m14 mova [cq+128*1], m15 mova [cq+128*2], m16 mova [cq+128*3], m17 mova [cq+128*4], m18 mova [cq+128*5], m19 mova [cq+128*6], m20 mova [cq+128*7], m21 jmp m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2 .pass2_end: psubsw m9, m0, m29 ; out31 paddsw m0, m29 ; out0 psubsw m29, m1, m28 ; out30 paddsw m1, m28 ; out1 psubsw m28, m2, m27 ; out29 paddsw m2, m27 ; out2 psubsw m27, m3, m26 ; out28 paddsw m3, m26 ; out3 psubsw m26, m4, m25 ; out27 paddsw m4, m25 ; out4 psubsw m25, m5, m24 ; out26 paddsw m5, m24 ; out5 psubsw m24, m6, m23 ; out25 paddsw m6, m23 ; out6 psubsw m23, m7, m22 ; out24 paddsw m7, m22 ; out7 call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8_start mova m0, [cq+128*0] mova m1, [cq+128*1] mova m2, [cq+128*2] mova m3, [cq+128*3] mova m4, [cq+128*4] mova m5, [cq+128*5] mova m6, [cq+128*6] mova m7, [cq+128*7] psubsw m22, m0, m21 ; out23 paddsw m0, m21 ; out8 psubsw m21, m1, m20 ; out22 paddsw m1, m20 ; out9 psubsw m20, m2, m19 ; out21 paddsw m2, m19 ; out10 psubsw m19, m3, m18 ; out20 paddsw m3, m18 ; out11 psubsw m18, m4, m17 ; out19 paddsw m4, m17 ; out12 psubsw m17, m5, m16 ; out18 paddsw m5, m16 ; out13 psubsw m16, m6, m15 ; out17 paddsw m6, m15 ; out14 psubsw m15, m7, m14 ; out16 paddsw m7, m14 ; out15 call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8 pmulhrsw m0, m11, m15 pmulhrsw m1, m11, m16 pmulhrsw m2, m11, m17 pmulhrsw m3, m11, m18 call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 pmulhrsw m0, m11, m19 pmulhrsw m1, m11, m20 pmulhrsw m2, m11, m21 pmulhrsw m3, m11, m22 call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 pmulhrsw m0, m11, m23 pmulhrsw m1, m11, m24 pmulhrsw m2, m11, m25 pmulhrsw m3, m11, m26 call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 pmulhrsw m0, m11, m27 pmulhrsw m1, m11, m28 pmulhrsw m2, m11, m29 pmulhrsw m3, m11, m9 jmp m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 32 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_32x16_10bpc).dconly2 .pass1_fast: mova m0, [cq+128* 0] mova m1, [cq+128* 4] mova m2, [cq+128* 8] mova m3, [cq+128*12] mov r6d, 16*12 call m(idct_8x16_internal_10bpc).main_fast mova m16, [cq+128* 2] mova m17, [cq+128* 6] mova m18, [cq+128*10] mova m19, [cq+128*14] call m(idct_16x16_internal_10bpc).main_fast call .pass1_load_spill call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast jmp .pass1_end .pass1: mova m0, [cq+128* 0] mova m1, [cq+128* 4] mova m2, [cq+128* 8] mova m3, [cq+128*12] mova m4, [cq+128*16] mova m5, [cq+128*20] mova m6, [cq+128*24] mova m7, [cq+128*28] call m(idct_8x16_internal_10bpc).main mova m16, [cq+128* 2] mova m17, [cq+128* 6] mova m18, [cq+128*10] mova m19, [cq+128*14] mova m20, [cq+128*18] mova m21, [cq+128*22] mova m22, [cq+128*26] mova m23, [cq+128*30] call m(idct_16x16_internal_10bpc).main call .pass1_load_spill mova m16, [cq+128*17] mova m17, [cq+128*19] mova m18, [cq+128*21] mova m19, [cq+128*23] mova m20, [cq+128*25] mova m21, [cq+128*27] mova m22, [cq+128*29] mova m23, [cq+128*31] call m(inv_txfm_add_dct_dct_32x16_10bpc).main .pass1_end: vpbroadcastd m11, [o(pd_2)] lea r4, [cq+128*8] call m(inv_txfm_add_dct_dct_32x16_10bpc).idct32_pass1_end punpckhqdq m22, m0, m20 ; 1 punpcklqdq m0, m20 ; 0 punpckhqdq m24, m2, m1 ; 5 punpcklqdq m1, m2, m1 ; 4 punpcklqdq m2, m14, m18 ; 8 punpckhqdq m26, m14, m18 ; 9 punpcklqdq m14, m15, m4 ; 2 punpckhqdq m23, m15, m4 ; 3 punpckhqdq m25, m3, m21 ; 7 punpcklqdq m15, m3, m21 ; 6 punpckhqdq m28, m6, m17 ; 13 punpcklqdq m3, m6, m17 ; 12 punpckhqdq m27, m5, m16 ; 11 punpcklqdq m16, m5, m16 ; 10 punpckhqdq m29, m7, m8 ; 15 punpcklqdq m17, m7, m8 ; 14 ret .pass1_load_spill: call m(inv_txfm_add_dct_dct_32x16_10bpc).idct16_sumsub mova [cq+128* 0], m0 mova m0, [cq+128* 1] mova [cq+128* 1], m1 mova [cq+128* 2], m2 mova m1, [cq+128* 3] mova m2, [cq+128* 5] mova [cq+128* 3], m3 mova [cq+128* 4], m4 mova m3, [cq+128* 7] mova m4, [cq+128* 9] mova [cq+128* 5], m5 mova [cq+128* 6], m6 mova [cq+128* 7], m7 mova m5, [cq+128*11] mova m6, [cq+128*13] mova m7, [cq+128*15] mova [cq+128* 8], m23 mova [cq+128* 9], m22 mova [cq+128*10], m21 mova [cq+128*11], m20 mova [cq+128*12], m19 mova [cq+128*13], m18 mova [cq+128*14], m17 mova [cq+128*15], m16 ret cglobal inv_txfm_add_identity_identity_32x32_10bpc, 4, 7, 16, dst, stride, c, eob %undef cmp vpbroadcastd m13, [pw_8192] vpbroadcastd m15, [pixel_10bpc_max] pxor m14, m14 lea r6, [strideq*9] cmp eobd, 136 jl .main mov r4, dstq call .main add cq, 64-128*4 lea dstq, [dstq+strideq*8] call .main add cq, 128*12-64 lea dstq, [r4+32] cmp eobd, 543 jl .main call .main add cq, 64-128*4 lea dstq, [dstq+strideq*8] .main: call .main_internal add cq, 128*4 pmulhrsw m1, m13, m2 pmulhrsw m3, m13, m4 pmulhrsw m5, m13, m6 pmulhrsw m7, m13, m8 call .main_internal jmp m(inv_txfm_add_identity_identity_16x32_10bpc).main2 .main_internal: mova m8, [cq+128* 0] packssdw m8, [cq+128* 8] mova m6, [cq+128* 1] packssdw m6, [cq+128* 9] mova m0, [cq+128* 2] packssdw m0, [cq+128*10] mova m2, [cq+128* 3] packssdw m2, [cq+128*11] REPX {vpermq x, x, q3120}, m8, m6, m0, m2 REPX {mova [cq+128*x], m14}, 0, 1, 2, 3 punpcklwd m4, m8, m6 punpckhwd m8, m6 punpcklwd m6, m0, m2 punpckhwd m0, m2 REPX {mova [cq+128*x], m14}, 8, 9, 10, 11 punpckldq m2, m4, m6 ; 0 1 punpckhdq m4, m6 ; 2 3 punpckldq m6, m8, m0 ; 4 5 punpckhdq m8, m0 ; 6 7 ret cglobal inv_txfm_add_dct_dct_16x64_10bpc, 4, 7, 0, dst, stride, c, eob lea r5, [o_base] test eobd, eobd jz .dconly PROLOGUE 4, 7, 32, -8*mmsize, dst, stride, c, eob %undef cmp vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] cmp eobd, 36 jl .fast call .pass1 cmp eobd, 151 jge .full lea r5, [o_base_8bpc] punpckhwd m22, m0, m0 punpckhwd m23, m1, m1 punpckhwd m24, m2, m2 punpckhwd m25, m3, m3 punpckhwd m26, m4, m4 punpckhwd m27, m5, m5 punpckhwd m28, m6, m6 punpckhwd m29, m7, m7 punpcklwd m21, m1, m1 punpcklwd m14, m3, m3 punpcklwd m18, m5, m5 punpcklwd m15, m7, m7 pxor m9, m9 punpcklwd m9, m9, m0 punpcklwd m8, m2, m2 punpcklwd m7, m4, m4 punpcklwd m1, m6, m6 call m(idct_16x16_internal_8bpc).main_fast2 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 mova [rsp+mmsize*0], m14 mova [rsp+mmsize*1], m15 mova [rsp+mmsize*2], m16 mova [rsp+mmsize*3], m17 mova [rsp+mmsize*4], m18 mova [rsp+mmsize*5], m19 mova [rsp+mmsize*6], m20 mova [rsp+mmsize*7], m21 call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast pxor m12, m12 mov r3d, 64*3 .zero_loop: REPX {mova [cq+r3*8+128*x], m12}, 0, 1, 2, 3 sub r3d, 64 jge .zero_loop jmp .pass2_end .full: mova [cq+128*0], m0 mova [cq+128*1], m1 mova [cq+128*2], m2 mova [cq+128*3], m3 mova [cq+128*4], m4 mova [cq+128*5], m5 mova [cq+128*6], m6 mova [cq+128*7], m7 add cq, 64 call .pass1 sub cq, 64 mova m22, [cq+128*0] ; 0 1 mova m23, [cq+128*1] ; 2 3 mova m24, [cq+128*2] ; 4 5 mova m25, [cq+128*3] ; 6 7 mova m26, [cq+128*4] ; 8 9 mova m27, [cq+128*5] ; 10 11 mova m28, [cq+128*6] ; 12 13 mova m29, [cq+128*7] ; 14 15 mova [cq+64* 8], m0 mova [cq+64* 9], m1 mova [cq+64*10], m2 mova [cq+64*11], m3 mova [cq+64*12], m4 mova [cq+64*13], m5 mova [cq+64*14], m6 mova [cq+64*15], m7 lea r5, [o_base_8bpc] punpcklwd m20, m1, m1 punpcklwd m16, m3, m3 punpcklwd m19, m5, m5 punpcklwd m17, m7, m7 punpcklwd m8, m24, m24 ; 4 punpcklwd m5, m2, m2 ; 20 punpcklwd m1, m28, m28 ; 12 punpcklwd m7, m26, m26 ; 8 punpcklwd m3, m4, m4 ; 24 punpcklwd m4, m6, m6 ; 28 pxor m9, m9 punpcklwd m6, m9, m0 ; __ 16 mova m0, m4 punpcklwd m9, m9, m22 ; __ 0 call m(idct_16x16_internal_8bpc).main_fast punpcklwd m21, m23, m23 ; 2 punpcklwd m15, m29, m29 ; 14 punpcklwd m18, m27, m27 ; 10 punpcklwd m14, m25, m25 ; 6 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast mova [rsp+mmsize*0], m14 mova [rsp+mmsize*1], m15 mova [rsp+mmsize*2], m16 mova [rsp+mmsize*3], m17 mova [rsp+mmsize*4], m18 mova [rsp+mmsize*5], m19 mova [rsp+mmsize*6], m20 mova [rsp+mmsize*7], m21 mova m21, [cq+64*15] mova m14, [cq+64* 8] mova m17, [cq+64*11] mova m18, [cq+64*12] mova m19, [cq+64*13] mova m16, [cq+64*10] mova m15, [cq+64* 9] mova m20, [cq+64*14] REPX {punpckhwd x, x}, m22, m21, m14, m29, m26, m17, m18, m25, \ m24, m19, m16, m27, m28, m15, m20, m23 call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf pxor m12, m12 mov r3d, 32*7 .full_zero_loop: REPX {mova [cq+r3*8+64*x], m12}, 0, 1, 2, 3 sub r3d, 32 jge .full_zero_loop jmp .pass2_end .fast: mova ym0, [cq+128*0] mova ym2, [cq+128*4] movshdup m8, [o(permB)] mova ym1, [cq+128*2] mova ym3, [cq+128*6] mova ym4, [cq+128*1] mova ym5, [cq+128*3] mova ym6, [cq+128*5] mova ym7, [cq+128*7] vpermt2q m0, m8, m2 ; 0 4 vpermt2q m1, m8, m3 ; 2 6 vpermt2q m4, m8, m5 ; 1 3 vpermt2q m7, m8, m6 ; 7 5 call m(idct_8x8_internal_10bpc).main_fast call m(idct_16x8_internal_10bpc).main_fast vpbroadcastd m11, [o(pd_2)] call m(idct_8x16_internal_10bpc).main_end2 mova m8, [o(idct8x32p)] packssdw m0, m4 packssdw m1, m5 packssdw m2, m6 packssdw m3, m7 mova m6, [dup16_perm] vpermb m0, m8, m0 vpermb m2, m8, m2 vprold m8, 16 vpermb m1, m8, m1 vpermb m3, m8, m3 punpckldq m4, m0, m2 punpckhdq m0, m2 punpckldq m2, m1, m3 punpckhdq m1, m3 punpckldq m21, m4, m2 punpckhdq m14, m4, m2 punpckldq m18, m0, m1 punpckhdq m15, m0, m1 vpord m7, m6, [o(pb_32)] {1to16} vpermb m22, m7, m21 ; 1 pmovzxwd m9, ym21 ; 0 vpermb m8, m6, m18 ; 4 vpermb m24, m7, m18 ; 5 vpermb m21, m6, m14 ; 2 vpermb m23, m7, m14 ; 3 vpermb m14, m6, m15 ; 6 vpermb m25, m7, m15 ; 7 lea r5, [o_base_8bpc] pslld m9, 16 pxor m7, m7 REPX {mova x, m7}, m1, m18, m15, m26, m27, m28, m29 call m(idct_16x16_internal_8bpc).main_fast2 call m(inv_txfm_add_dct_dct_16x32_8bpc).main_oddhalf_fast2 mova [rsp+mmsize*0], m14 mova [rsp+mmsize*1], m15 mova [rsp+mmsize*2], m16 mova [rsp+mmsize*3], m17 mova [rsp+mmsize*4], m18 mova [rsp+mmsize*5], m19 mova [rsp+mmsize*6], m20 mova [rsp+mmsize*7], m21 call m(inv_txfm_add_dct_dct_16x64_8bpc).main_oddhalf_fast pxor m12, m12 REPX {mova [cq+128*x], ym12}, 0, 1, 2, 3, 4, 5, 6, 7 .pass2_end: movshdup m30, [permC] vpbroadcastd m11, [pw_2048] vpbroadcastd m13, [pixel_10bpc_max] lea r6, [strideq*3] psrlq m31, m30, 8 vpermq m8, m30, m0 vpermq m9, m31, m1 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m2 vpermq m9, m31, m3 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m4 vpermq m9, m31, m5 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m6 vpermq m9, m31, m7 call m(idct_16x8_internal_10bpc).write_16x4 mova m1, [rsp+mmsize*0] mova m2, [rsp+mmsize*1] mova m3, [rsp+mmsize*2] mova m4, [rsp+mmsize*3] mova m5, [rsp+mmsize*4] mova m6, [rsp+mmsize*5] mova m7, [rsp+mmsize*6] mova m8, [rsp+mmsize*7] paddsw m0, m1, m21 psubsw m21, m1, m21 paddsw m1, m2, m20 psubsw m20, m2, m20 paddsw m2, m3, m19 psubsw m19, m3, m19 paddsw m3, m4, m18 psubsw m18, m4, m18 paddsw m4, m5, m17 psubsw m17, m5, m17 paddsw m5, m6, m16 psubsw m16, m6, m16 paddsw m6, m7, m15 psubsw m15, m7, m15 paddsw m7, m8, m14 psubsw m14, m8, m14 vpermq m8, m30, m0 vpermq m9, m31, m1 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m2 vpermq m9, m31, m3 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m4 vpermq m9, m31, m5 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m6 vpermq m9, m31, m7 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m14 vpermq m9, m31, m15 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m16 vpermq m9, m31, m17 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m18 vpermq m9, m31, m19 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m20 vpermq m9, m31, m21 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m22 vpermq m9, m31, m23 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m24 vpermq m9, m31, m25 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m26 vpermq m9, m31, m27 call m(idct_16x8_internal_10bpc).write_16x4 vpermq m8, m30, m28 vpermq m9, m31, m29 call m(idct_16x8_internal_10bpc).write_16x4 RET .pass1: mova m0, [cq+128* 0] mova m1, [cq+128* 2] mova m2, [cq+128* 4] mova m3, [cq+128* 6] mova m4, [cq+128* 8] mova m5, [cq+128*10] mova m6, [cq+128*12] mova m7, [cq+128*14] call m(idct_8x16_internal_10bpc).main mova m16, [cq+128* 1] mova m17, [cq+128* 3] mova m18, [cq+128* 5] mova m19, [cq+128* 7] mova m20, [cq+128* 9] mova m21, [cq+128*11] mova m22, [cq+128*13] mova m23, [cq+128*15] call m(idct_16x16_internal_10bpc).main call m(idct_16x16_internal_10bpc).main_end jmp m(idct_16x16_internal_10bpc).main_end3 .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 64 add r6d, 640 sar r6d, 10 jmp m(inv_txfm_add_dct_dct_16x8_10bpc).dconly2 cglobal inv_txfm_add_dct_dct_32x64_10bpc, 4, 7, 0, dst, stride, c, eob lea r5, [o_base] test eobd, eobd jz .dconly PROLOGUE 4, 7, 32, -64*40, dst, stride, c, eob %undef cmp vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] cmp eobd, 136 jl .fast add cq, 64 cmp eobd, 543 jge .full call .pass1_fast ; bottomright 16x16 zero jmp .lefthalf .full: call .pass1 mov r3d, 16*28 .lefthalf: mova [cq+128* 0], m27 mova [cq+128* 1], m14 mova [cq+128* 2], m28 mova [cq+128* 3], m15 mova [cq+128* 4], m22 mova [cq+128* 5], m23 mova [cq+128* 6], m24 mova [cq+128* 7], m25 mova [cq+128* 8], m0 mova [cq+128* 9], m26 mova [cq+128*10], m20 mova [cq+128*11], m21 mova [cq+128*12], m18 mova [cq+128*13], m16 mova [cq+128*14], m17 mova [cq+128*15], m3 sub cq, 64 vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] call .pass1 call .pass2_start pxor m31, m31 .right_zero_loop: REPX {mova [cq+r3*8+64+128*x], m31}, 0, 1, 2, 3 sub r3d, 16*4 jge .right_zero_loop mov r3d, 16*28 jmp .left_zero_loop .pass2_start: vpbroadcastd m10, [o(pd_2048)] lea r5, [o_base_8bpc] lea r4, [rsp+gprsize] mova m1, [cq+128*15+64] mova m2, [cq+128* 8+64] call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 mova m0, m21 mova m1, [cq+128*12+64] mova m2, [cq+128*11+64] mova m3, m18 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 mova m0, m20 mova m1, [cq+128*13+64] mova m2, [cq+128*10+64] mova m3, m16 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 mova m0, m26 mova m1, [cq+128*14+64] mova m2, [cq+128* 9+64] mova m3, m17 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 mova m0, m27 mova m1, m28 mova m2, [cq+128* 0+64] mova m3, [cq+128* 2+64] mova m16, [cq+128* 1+64] mova m17, [cq+128* 3+64] call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast mova m26, [cq+128* 4+64] mova m27, [cq+128* 5+64] mova m28, [cq+128* 6+64] mova m29, [cq+128* 7+64] mova [rsp+64*32+gprsize], m14 mova [rsp+64*33+gprsize], m15 mova [rsp+64*34+gprsize], m16 mova [rsp+64*35+gprsize], m17 mova [rsp+64*36+gprsize], m18 mova [rsp+64*37+gprsize], m19 mova [rsp+64*38+gprsize], m20 mova [rsp+64*39+gprsize], m21 jmp m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast .fast: ; topleft 16x16 nonzero cmp eobd, 36 jl .fast2 call .pass1_fast vpbroadcastd m10, [o(pd_2048)] call .pass2_fast_start jmp .end .fast2: ; topleft 8x8 nonzero movshdup m7, [o(permB)] mova ym0, [cq+128*0] mova ym1, [cq+128*4] mova ym4, [cq+128*2] mova ym5, [cq+128*6] mova ym16, [cq+128*1] mova ym2, [cq+128*5] mova ym3, [cq+128*3] mova ym17, [cq+128*7] mov r3d, 16*4 vpermq m0, m7, m0 ; 0 0 vpermq m1, m7, m1 ; 4 4 vpermt2q m4, m7, m5 ; 2 6 vpermt2q m16, m7, m2 ; 1 5 vpermt2q m17, m7, m3 ; 7 3 REPX {pmulld x, m12}, m0, m1, m4, m16, m17 REPX {paddd x, m13}, m0, m1, m4, m16, m17 REPX {psrad x, 12 }, m0, m1, m4, m16, m17 call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast2 vpbroadcastd m11, [o(pd_1)] call m(idct_16x16_internal_10bpc).main_end2 call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 punpcklqdq m27, m0, m2 ; 0 punpckhqdq m0, m2 ; 1 punpcklqdq m22, m3, m4 ; 2 punpckhqdq m26, m3, m4 ; 3 punpcklqdq m14, m5, m7 ; 4 punpckhqdq m20, m5, m7 ; 5 punpcklqdq m23, m6, m8 ; 6 punpckhqdq m21, m6, m8 ; 7 mova m10, m13 call .pass2_fast2_start .end: pxor m31, m31 .left_zero_loop: REPX {mova [cq+r3*8+128*x], m31}, 0, 1, 2, 3 sub r3d, 16*4 jge .left_zero_loop call .pass2_end RET .pass2_end: DEFINE_ARGS dst, stride, _, dst2, stride32, stklo, stkhi vpbroadcastd m30, [pixel_10bpc_max] vpbroadcastd m13, [pw_2048] mov stride32q, strideq shl stride32q, 5 lea stkhiq, [rsp+31*mmsize+gprsize] lea dst2q, [dstq+stride32q] lea stkloq, [rsp+gprsize] sub dst2q, strideq ; dst31 paddsw m8, m0, m29 ; t0[idct32] psubsw m9, m0, m29 ; t31[idct32] call .end_sumsub_write paddsw m8, m1, m28 ; t1[idct32] psubsw m9, m1, m28 ; t30[idct32] call .end_sumsub_write paddsw m8, m2, m27 ; t2[idct32] psubsw m9, m2, m27 ; t29[idct32] call .end_sumsub_write paddsw m8, m3, m26 ; t3[idct32] psubsw m9, m3, m26 ; t28[idct32] call .end_sumsub_write paddsw m8, m4, m25 ; t4[idct32] psubsw m9, m4, m25 ; t27[idct32] call .end_sumsub_write paddsw m8, m5, m24 ; t5[idct32] psubsw m9, m5, m24 ; t26[idct32] call .end_sumsub_write paddsw m8, m6, m23 ; t6[idct32] psubsw m9, m6, m23 ; t25[idct32] call .end_sumsub_write paddsw m8, m7, m22 ; t7[idct32] psubsw m9, m7, m22 ; t24[idct32] call .end_sumsub_write mova m0, [rsp+64*32+gprsize] mova m1, [rsp+64*33+gprsize] mova m2, [rsp+64*34+gprsize] mova m3, [rsp+64*35+gprsize] mova m4, [rsp+64*36+gprsize] mova m5, [rsp+64*37+gprsize] mova m6, [rsp+64*38+gprsize] mova m7, [rsp+64*39+gprsize] paddsw m8, m0, m21 ; t8[idct32] psubsw m9, m0, m21 ; t23[idct32] call .end_sumsub_write paddsw m8, m1, m20 ; t9[idct32] psubsw m9, m1, m20 ; t22[idct32] call .end_sumsub_write paddsw m8, m2, m19 ; t10[idct32] psubsw m9, m2, m19 ; t21[idct32] call .end_sumsub_write paddsw m8, m3, m18 ; t11[idct32] psubsw m9, m3, m18 ; t20[idct32] call .end_sumsub_write paddsw m8, m4, m17 ; t12[idct32] psubsw m9, m4, m17 ; t19[idct32] call .end_sumsub_write paddsw m8, m5, m16 ; t13[idct32] psubsw m9, m5, m16 ; t18[idct32] call .end_sumsub_write paddsw m8, m6, m15 ; t14[idct32] psubsw m9, m6, m15 ; t17[idct32] call .end_sumsub_write paddsw m8, m7, m14 ; t15[idct32] psubsw m9, m7, m14 ; t16[idct32] ; fall-through .end_sumsub_write: mova m10, [stkhiq] ; t63-n mova m12, [stkloq] ; t32+n psubsw m11, m8, m10 ; out63-n paddsw m8, m10 ; out0 +n psubsw m10, m9, m12 ; out32+n paddsw m9, m12 ; out32-n REPX {pmulhrsw x, m13}, m11, m8, m10, m9 paddw m8, [dstq] paddw m9, [dst2q] paddw m10, [dstq+stride32q] paddw m11, [dst2q+stride32q] REPX {pminsw x, m30}, m11, m8, m10, m9 REPX {pmaxsw x, m31}, m11, m8, m10, m9 mova [dstq ], m8 mova [dst2q ], m9 mova [dstq +stride32q], m10 mova [dst2q+stride32q], m11 add stkloq, mmsize sub stkhiq, mmsize add dstq, strideq sub dst2q, strideq ret .pass2_fast_start: lea r5, [o_base_8bpc] lea r4, [rsp+gprsize] call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast mova m0, m21 mova m3, m18 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast mova m0, m20 mova m3, m16 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast mova m0, m26 mova m3, m17 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 mova m0, m27 mova m1, m28 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast2 mova [rsp+64*32+gprsize], m14 mova [rsp+64*33+gprsize], m15 mova [rsp+64*34+gprsize], m16 mova [rsp+64*35+gprsize], m17 mova [rsp+64*36+gprsize], m18 mova [rsp+64*37+gprsize], m19 mova [rsp+64*38+gprsize], m20 mova [rsp+64*39+gprsize], m21 jmp m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast2 .pass2_fast2_start: lea r5, [o_base_8bpc] lea r4, [rsp+gprsize] call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast2 mova m0, m21 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast2 mova m0, m20 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast2 mova m0, m26 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part1_fast2 call m(inv_txfm_add_dct_dct_32x64_8bpc).main_part2 mova m0, m27 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast3 mova [rsp+64*32+gprsize], m14 mova [rsp+64*33+gprsize], m15 mova [rsp+64*34+gprsize], m16 mova [rsp+64*35+gprsize], m17 mova [rsp+64*36+gprsize], m18 mova [rsp+64*37+gprsize], m19 mova [rsp+64*38+gprsize], m20 mova [rsp+64*39+gprsize], m21 jmp m(inv_txfm_add_dct_dct_32x32_8bpc).main_oddhalf_fast3 .dconly: DEFINE_ARGS dst, stride, c, eob imul r6d, [cq], 181 mov [cq], eobd or r3d, 64 jmp m(inv_txfm_add_dct_dct_32x16_10bpc).dconly3 .pass1_fast: pmulld m0, m12, [cq+128* 0] pmulld m1, m12, [cq+128* 4] pmulld m2, m12, [cq+128* 8] pmulld m3, m12, [cq+128*12] mov r3d, 16*12 call m(idct_8x16_internal_10bpc).main_fast_rect2 pmulld m16, m12, [cq+128* 2] pmulld m17, m12, [cq+128* 6] pmulld m18, m12, [cq+128*10] pmulld m19, m12, [cq+128*14] call m(idct_16x16_internal_10bpc).main_fast_rect2 call .pass1_load_spill call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast_rect2 jmp .pass1_end .pass1: pmulld m0, m12, [cq+128* 0] pmulld m1, m12, [cq+128* 4] pmulld m2, m12, [cq+128* 8] pmulld m3, m12, [cq+128*12] pmulld m4, m12, [cq+128*16] pmulld m5, m12, [cq+128*20] pmulld m6, m12, [cq+128*24] pmulld m7, m12, [cq+128*28] call m(idct_8x16_internal_10bpc).main_rect2 pmulld m16, m12, [cq+128* 2] pmulld m17, m12, [cq+128* 6] pmulld m18, m12, [cq+128*10] pmulld m19, m12, [cq+128*14] pmulld m20, m12, [cq+128*18] pmulld m21, m12, [cq+128*22] pmulld m22, m12, [cq+128*26] pmulld m23, m12, [cq+128*30] call m(idct_16x16_internal_10bpc).main_rect2 call .pass1_load_spill pmulld m16, m12, [cq+128*17] pmulld m17, m12, [cq+128*19] pmulld m18, m12, [cq+128*21] pmulld m19, m12, [cq+128*23] pmulld m20, m12, [cq+128*25] pmulld m21, m12, [cq+128*27] pmulld m22, m12, [cq+128*29] pmulld m23, m12, [cq+128*31] call m(inv_txfm_add_dct_dct_32x16_10bpc).main_rect2 .pass1_end: vpbroadcastd m11, [o(pd_1)] lea r4, [cq+128*8] call m(inv_txfm_add_dct_dct_32x16_10bpc).idct32_pass1_end punpcklqdq m27, m0, m20 ; 0 punpckhqdq m0, m20 ; 1 punpcklqdq m24, m5, m16 ; 10 punpckhqdq m16, m5, m16 ; 11 punpcklqdq m23, m3, m21 ; 6 punpckhqdq m21, m3, m21 ; 7 punpcklqdq m25, m7, m8 ; 14 punpckhqdq m3, m7, m8 ; 15 punpcklqdq m22, m15, m4 ; 2 punpckhqdq m26, m15, m4 ; 3 punpcklqdq m15, m6, m17 ; 12 punpckhqdq m17, m6, m17 ; 13 punpcklqdq m28, m14, m18 ; 8 punpckhqdq m18, m14, m18 ; 9 punpcklqdq m14, m2, m1 ; 4 punpckhqdq m20, m2, m1 ; 5 ret .pass1_load_spill: call m(inv_txfm_add_dct_dct_32x16_10bpc).idct16_sumsub mova [cq+128* 0], m0 pmulld m0, m12, [cq+128* 1] mova [cq+128* 1], m1 mova [cq+128* 2], m2 pmulld m1, m12, [cq+128* 3] pmulld m2, m12, [cq+128* 5] mova [cq+128* 3], m3 mova [cq+128* 4], m4 pmulld m3, m12, [cq+128* 7] pmulld m4, m12, [cq+128* 9] mova [cq+128* 5], m5 mova [cq+128* 6], m6 mova [cq+128* 7], m7 pmulld m5, m12, [cq+128*11] pmulld m6, m12, [cq+128*13] pmulld m7, m12, [cq+128*15] mova [cq+128* 8], m23 mova [cq+128* 9], m22 mova [cq+128*10], m21 mova [cq+128*11], m20 mova [cq+128*12], m19 mova [cq+128*13], m18 mova [cq+128*14], m17 mova [cq+128*15], m16 ret cglobal inv_txfm_add_dct_dct_64x16_10bpc, 4, 7, 0, dst, stride, c, eob %undef cmp lea r5, [o_base] test eobd, eobd jz .dconly PROLOGUE 4, 7, 32, -64*32, dst, stride, c, eob %undef cmp vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] cmp eobd, 36 jl .fast ; 8x8 cmp eobd, 151 jge .full ; 16x16 lea r4, [idct64_mul_16bpc] lea r6, [rsp+4*64] mova m0, [cq+64* 1] mova m3, [cq+64*15] call .main_part1_fast mova m0, [cq+64* 7] mova m3, [cq+64* 9] call .main_part1_fast mova m0, [cq+64* 5] mova m3, [cq+64*11] call .main_part1_fast mova m0, [cq+64* 3] mova m3, [cq+64*13] call .main_part1_fast call .main_part2 mova m0, [cq+64* 0] mova m1, [cq+64* 8] mova m16, [cq+64* 4] mova m17, [cq+64*12] call m(idct_8x16_internal_10bpc).main_fast2 call m(idct_16x16_internal_10bpc).main_fast2 call m(inv_txfm_add_dct_dct_32x16_10bpc).idct16_sumsub call .pass1_load_spill call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast2 mov r6d, 12*8 jmp .idct64_end .full: lea r4, [idct64_mul_16bpc] lea r6, [rsp+4*64] mova m0, [cq+64* 1] mova m1, [cq+64*31] mova m2, [cq+64*17] mova m3, [cq+64*15] call .main_part1 mova m0, [cq+64* 7] mova m1, [cq+64*25] mova m2, [cq+64*23] mova m3, [cq+64* 9] call .main_part1 mova m0, [cq+64* 5] mova m1, [cq+64*27] mova m2, [cq+64*21] mova m3, [cq+64*11] call .main_part1 mova m0, [cq+64* 3] mova m1, [cq+64*29] mova m2, [cq+64*19] mova m3, [cq+64*13] call .main_part1 call .main_part2 mova m0, [cq+64* 0] mova m1, [cq+64* 8] mova m2, [cq+64*16] mova m3, [cq+64*24] mova m16, [cq+64* 4] mova m17, [cq+64*12] mova m18, [cq+64*20] mova m19, [cq+64*28] call m(idct_8x16_internal_10bpc).main_fast call m(idct_16x16_internal_10bpc).main_fast call m(inv_txfm_add_dct_dct_32x16_10bpc).idct16_sumsub call .pass1_load_spill mova m4, [cq+64*18] mova m5, [cq+64*22] mova m6, [cq+64*26] mova m7, [cq+64*30] call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast mov r6d, 28*8 jmp .idct64_end .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 16 .dconly1: add r6d, 640 sar r6d, 10 .dconly2: vpbroadcastd m3, [o(dconly_10bpc)] imul r6d, 181 add r6d, 2176 sar r6d, 12 vpbroadcastw m2, r6d paddsw m2, m3 .dconly_loop: paddsw m0, m2, [dstq+64*0] paddsw m1, m2, [dstq+64*1] psubusw m0, m3 psubusw m1, m3 mova [dstq+64*0], m0 mova [dstq+64*1], m1 add dstq, strideq dec r3d jg .dconly_loop ret .pass1_load_spill: mova [cq+64* 0], m0 mova m0, [cq+64* 2] mova [cq+64* 2], m1 mova m1, [cq+64* 6] mova [cq+64* 4], m2 mova [cq+64* 6], m3 mova m2, [cq+64*10] mova m3, [cq+64*14] mova [cq+64* 8], m4 mova [cq+64*10], m5 mova [cq+64*12], m6 mova [cq+64*14], m7 mova [cq+64* 1], m23 mova [cq+64* 3], m22 mova [cq+64* 5], m21 mova [cq+64* 7], m20 mova [cq+64* 9], m19 mova [cq+64*11], m18 mova [cq+64*13], m17 mova [cq+64*15], m16 ret ALIGN function_align .main_part1_fast_rect2: REPX {paddd x, m13}, m0, m3 REPX {psrad x, 12 }, m0, m3 .main_part1_fast: pmulld m7, m0, [r4+4*0]{bcstd} ; t63a pmulld m0, [r4+4*1]{bcstd} ; t32a pmulld m4, m3, [r4+4*6]{bcstd} ; t60a pmulld m3, [r4+4*7]{bcstd} ; t35a vpbroadcastd m10, [r4+4*8] vpbroadcastd m11, [r4+4*9] REPX {paddd x, m13}, m7, m0, m4, m3 REPX {psrad x, 12 }, m7, m0, m4, m3 mova m8, m0 mova m1, m7 mova m6, m3 mova m2, m4 jmp .main_part1b .main_part1_rect2: REPX {paddd x, m13}, m0, m1, m2, m3 REPX {psrad x, 12 }, m0, m1, m2, m3 .main_part1: ; idct64 steps 1-5 ; in1/31/17/15 -> t32a/33/34a/35/60/61a/62/63a ; in7/25/23/ 9 -> t56a/57/58a/59/36/37a/38/39a ; in5/27/21/11 -> t40a/41/42a/43/52/53a/54/55a ; in3/29/19/13 -> t48a/49/50a/51/44/45a/46/47a pmulld m7, m0, [r4+4*0]{bcstd} ; t63a pmulld m0, [r4+4*1]{bcstd} ; t32a pmulld m6, m1, [r4+4*2]{bcstd} ; t62a pmulld m1, [r4+4*3]{bcstd} ; t33a pmulld m5, m2, [r4+4*4]{bcstd} ; t61a pmulld m2, [r4+4*5]{bcstd} ; t34a pmulld m4, m3, [r4+4*6]{bcstd} ; t60a pmulld m3, [r4+4*7]{bcstd} ; t35a vpbroadcastd m10, [r4+4*8] vpbroadcastd m11, [r4+4*9] REPX {paddd x, m13}, m7, m0, m6, m1, m5, m2, m4, m3 REPX {psrad x, 12 }, m0, m1, m7, m6, m2, m3, m5, m4 psubd m8, m0, m1 ; t33 paddd m0, m1 ; t32 psubd m1, m7, m6 ; t62 paddd m7, m6 ; t63 psubd m6, m3, m2 ; t34 paddd m3, m2 ; t35 psubd m2, m4, m5 ; t61 paddd m4, m5 ; t60 .main_part1b: REPX {pmaxsd x, m14}, m8, m1, m6, m2 REPX {pminsd x, m15}, m8, m1, m6, m2 ITX_MULSUB_2D 1, 8, 5, 9, _, 13, 10, 11 ; t33a, t62a ITX_MULSUB_2D 2, 6, 5, 9, _, 13, 10, 11, 2 ; t61a, t34a REPX {pmaxsd x, m14}, m0, m3, m7, m4 REPX {pminsd x, m15}, m0, m3, m7, m4 vpbroadcastd m10, [r4+4*10] vpbroadcastd m11, [r4+4*11] psubd m5, m0, m3 ; t35a paddd m0, m3 ; t32a psubd m3, m7, m4 ; t60a paddd m7, m4 ; t63a psubd m4, m1, m6 ; t34 paddd m1, m6 ; t33 psubd m6, m8, m2 ; t61 paddd m8, m2 ; t62 REPX {pmaxsd x, m14}, m5, m3, m4, m6 REPX {pminsd x, m15}, m5, m3, m4, m6 ITX_MULSUB_2D 3, 5, 2, 9, _, 13, 10, 11 ; t35, t60 ITX_MULSUB_2D 6, 4, 2, 9, _, 13, 10, 11 ; t34a, t61a REPX {pmaxsd x, m14}, m0, m7, m1, m8 REPX {pminsd x, m15}, m0, m7, m1, m8 add r4, 4*12 mova [r6-64*4], m0 mova [r6+64*3], m7 mova [r6-64*3], m1 mova [r6+64*2], m8 mova [r6-64*2], m6 mova [r6+64*1], m4 mova [r6-64*1], m3 mova [r6+64*0], m5 add r6, 64*8 ret .main_part2: ; idct64 steps 6-9 lea r4, [r6+64*3] sub r6, 64*4 vpbroadcastd m10, [pd_1567] vpbroadcastd m11, [pd_3784] .main_part2_loop: mova m0, [r6-64*32] ; t32a mova m1, [r4-64*24] ; t39a mova m2, [r4-64*32] ; t63a mova m3, [r6-64*24] ; t56a mova m4, [r6-64*16] ; t40a mova m5, [r4-64* 8] ; t47a mova m6, [r4-64*16] ; t55a mova m7, [r6-64* 8] ; t48a psubd m8, m0, m1 ; t39 paddd m0, m1 ; t32 psubd m1, m2, m3 ; t56 paddd m2, m3 ; t63 psubd m3, m5, m4 ; t40 paddd m5, m4 ; t47 psubd m4, m7, m6 ; t55 paddd m7, m6 ; t48 REPX {pmaxsd x, m14}, m8, m1, m3, m4 REPX {pminsd x, m15}, m8, m1, m3, m4 ITX_MULSUB_2D 1, 8, 6, 9, _, 13, 10, 11 ; t39a, t56a ITX_MULSUB_2D 4, 3, 6, 9, _, 13, 10, 11, 2 ; t55a, t40a REPX {pmaxsd x, m14}, m0, m2, m5, m7 REPX {pminsd x, m15}, m0, m5, m2, m7 psubd m6, m2, m7 ; t48a paddd m2, m7 ; t63a psubd m7, m0, m5 ; t47a paddd m0, m5 ; t32a psubd m5, m8, m4 ; t55 paddd m8, m4 ; t56 psubd m4, m1, m3 ; t40 paddd m1, m3 ; t39 REPX {pmaxsd x, m14}, m6, m7, m5, m4 REPX {pminsd x, m15}, m6, m7, m5, m4 REPX {pmulld x, m12}, m6, m7, m5, m4 REPX {pmaxsd x, m14}, m2, m0, m8, m1 REPX {pminsd x, m15}, m2, m0, m8, m1 paddd m6, m13 paddd m5, m13 psubd m3, m6, m7 ; t47 paddd m6, m7 ; t48 psubd m7, m5, m4 ; t40a paddd m5, m4 ; t55a REPX {psrad x, 12}, m3, m6, m7, m5 mova [r4-64* 8], m2 mova [r6-64*32], m0 mova [r6-64* 8], m8 mova [r4-64*32], m1 mova [r4-64*24], m3 mova [r6-64*16], m6 mova [r6-64*24], m7 mova [r4-64*16], m5 add r6, 64 sub r4, 64 cmp r6, r4 jl .main_part2_loop ret .idct64_main_end: %macro IDCT64_PASS1_END 9 mova m%5, [%9+%1*128] ; t0+n [idct32] + idct64 rounding psubd m%6, m%5, m%2 ; out31-n [idct32] = t31-n [idct64] paddd m%5, m%2 ; out0+n [idct32] = t0+n [idct64] REPX {pmaxsd x, m14}, m%6, m%5 REPX {pminsd x, m15}, m%6, m%5 REPX {paddd x, m11}, m%6, m%5 mova m%2, [r3+%3*64] ; t32+n [idct64] mova m%7, [r3+%4*64] ; t63-n [idct64] psubd m%8, m%5, m%7 ; out63-n paddd m%5, m%7 ; out0+n psubd m%7, m%6, m%2 ; out32+n paddd m%6, m%2 ; out31-n REPX {vpsravd x, m11}, m%8, m%5, m%7, m%6 %endmacro %macro IDCT64_PASS1_ENDx4 1 %assign %%m1 %1 ; t32+n %assign %%m2 (7-%1) ; t39-n %assign %%m3 (8+%1) ; t40+n %assign %%m4 (15-%1) ; t47-n %assign %%m5 (16+%1) ; t48+n %assign %%m6 (23-%1) ; t55-n %assign %%m7 (24+%1) ; t56+n %assign %%m8 (31-%1) ; t63-n %assign %%r1 %1 ; t16+n %assign %%r2 (7-%1) ; t23-n %assign %%r3 (16+%1) ; t24-n %assign %%r4 (23-%1) ; t31-n %assign %%c1 (%1) ; t0/8+n %assign %%c2 (7-%1) ; t7/15-n IDCT64_PASS1_END %%c1, %%r4, %%m1, %%m8, 24, 25, 26, 27, cq ; out0/31/32/63 IDCT64_PASS1_END %%c1, %%r1, %%m4, %%m5, 28, 29, 30, 31, r4 ; out15/16/47/48 packssdw m %+ %%r1, m24, m29 packssdw m %+ %%r4, m28, m25 packssdw m26, m31 packssdw m30, m27 mova [r3+%%m5*mmsize], m26 mova [r3+%%m8*mmsize], m30 IDCT64_PASS1_END %%c2, %%r3, %%m2, %%m7, 24, 25, 26, 27, cq ; out7/24/39/56 IDCT64_PASS1_END %%c2, %%r2, %%m3, %%m6, 28, 29, 30, 31, r4 ; out8/23/40/55 packssdw m %+ %%r2, m24, m29 packssdw m %+ %%r3, m28, m25 packssdw m26, m31 packssdw m30, m27 mova [r3+%%m6*mmsize], m26 mova [r3+%%m7*mmsize], m30 %endmacro IDCT64_PASS1_ENDx4 0 IDCT64_PASS1_ENDx4 1 IDCT64_PASS1_ENDx4 2 IDCT64_PASS1_ENDx4 3 ret .idct64_end: vpbroadcastd m11, [o(pd_2)] lea r4, [cq+64] mov r3, rsp lea r5, [o_base_8bpc] call .idct64_main_end pxor m12, m12 .zero_loop: REPX {mova [cq+r6*8+64*x], m12}, 0, 1, 2, 3 sub r6d, 8*4 jge .zero_loop lea r3, [strideq*3] mov r4, dstq call .pass2 mova m0, [rsp+16*mmsize] mova m1, [rsp+17*mmsize] mova m2, [rsp+18*mmsize] mova m3, [rsp+19*mmsize] mova m4, [rsp+20*mmsize] mova m5, [rsp+21*mmsize] mova m6, [rsp+22*mmsize] mova m7, [rsp+23*mmsize] mova m16, [rsp+24*mmsize] mova m17, [rsp+25*mmsize] mova m18, [rsp+26*mmsize] mova m19, [rsp+27*mmsize] mova m20, [rsp+28*mmsize] mova m21, [rsp+29*mmsize] mova m22, [rsp+30*mmsize] mova m23, [rsp+31*mmsize] lea dstq, [r4+64] call .pass2 RET .pass2: psrlq m12, [permC], 24 ; 0 2 8 10 1 3 9 11 psrlq m13, m12, 32 ; 4 6 12 14 5 7 13 15 call m(inv_txfm_add_dct_dct_32x16_10bpc).transpose_16x32 punpckhqdq m19, m5, m16 ; 11 punpcklqdq m5, m16 ; 10 punpckhqdq m16, m2, m1 ; 5 punpcklqdq m2, m1 ; 4 punpcklqdq m1, m15, m4 ; 2 punpckhqdq m15, m4 ; 3 punpcklqdq m4, m14, m18 ; 8 punpckhqdq m18, m14, m18 ; 9 punpckhqdq m14, m0, m20 ; 1 punpcklqdq m0, m20 ; 0 punpckhqdq m20, m6, m17 ; 13 punpcklqdq m6, m17 ; 12 punpckhqdq m17, m3, m21 ; 7 punpcklqdq m3, m21 ; 6 punpckhqdq m21, m7, m8 ; 15 punpcklqdq m7, m8 ; 14 call m(inv_txfm_add_dct_dct_32x8_8bpc).main call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf .write: vpbroadcastd m11, [pw_2048] pxor m12, m12 vpbroadcastd m13, [pixel_10bpc_max] call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x8 pmulhrsw m0, m11, m14 pmulhrsw m1, m11, m15 pmulhrsw m2, m11, m16 pmulhrsw m3, m11, m17 call m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 pmulhrsw m0, m11, m18 pmulhrsw m1, m11, m19 pmulhrsw m2, m11, m20 pmulhrsw m3, m11, m21 jmp m(inv_txfm_add_dct_dct_32x8_10bpc).write_32x4 .fast: ; 8x8 packed movshdup m7, [o(permB)] mova ym0, [cq+64*1] mova ym2, [cq+64*5] mova ym3, [cq+64*3] mova ym1, [cq+64*7] vpermt2q m0, m7, m2 ; 1 5 vpermt2q m1, m7, m3 ; 7 3 call .main_oddhalf_packed mova [rsp+ 0*mmsize], m0 mova [rsp+ 1*mmsize], m1 mova [rsp+ 2*mmsize], m2 mova [rsp+ 3*mmsize], m3 mova [rsp+ 4*mmsize], m4 mova [rsp+ 5*mmsize], m5 mova [rsp+ 6*mmsize], m6 mova [rsp+ 7*mmsize], m7 mova [rsp+ 8*mmsize], m16 mova [rsp+ 9*mmsize], m17 mova [rsp+10*mmsize], m18 mova [rsp+11*mmsize], m19 mova [rsp+12*mmsize], m20 mova [rsp+13*mmsize], m21 mova [rsp+14*mmsize], m22 mova [rsp+15*mmsize], m23 movshdup m7, [o(permB)] mova ym0, [cq+64*0] mova ym4, [cq+64*4] mova ym16, [cq+64*2] mova ym5, [cq+64*6] vpermt2q m16, m7, m5 ; 2 6 vpermq m0, m7, m0 ; 0 0 vpermq m4, m7, m4 ; 4 4 call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast3 ; m0-7,9,16-22 contain un-sumsub'ed dct32 output data ; zero input coefs pxor m12, m12 REPX {mova [cq+x*64], ym12}, 0, 1, 2, 3, 4, 5, 6, 7 vpbroadcastd m11, [o(pd_2)] call .main_end lea r3, [strideq*3] mov r4, dstq call .pass2_fast mova m0, m24 mova m1, m25 mova m2, m26 mova m3, m27 mova m4, m28 mova m5, m29 mova m6, m30 mova m7, m31 lea dstq, [r4+64] lea r5, [o_base] call .pass2_fast RET .pass2_fast: call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 lea r5, [o_base_8bpc] punpckhqdq m14, m0, m2 ; 1 punpcklqdq m0, m2 ; 0 punpcklqdq m1, m3, m4 ; 2 punpckhqdq m15, m3, m4 ; 3 punpcklqdq m2, m5, m7 ; 4 punpckhqdq m16, m5, m7 ; 5 punpcklqdq m3, m6, m8 ; 6 punpckhqdq m17, m6, m8 ; 7 call m(inv_txfm_add_dct_dct_32x16_8bpc).main_oddhalf_fast jmp .write .main_end: %macro IDCT64_PASS1_PACKED_END 7 psubd m%5, m%1, m%2 ; out31-n [idct32] = t31-n [idct64] paddd m%1, m%2 ; out0+n [idct32] = t0+n [idct64] REPX {pmaxsd x, m14}, m%5, m%1 REPX {pminsd x, m15}, m%5, m%1 REPX {paddd x, m11}, m%5, m%1 mova m%2, [rsp+%6*64+gprsize] ; t32+n [idct64] mova m%3, [rsp+%7*64+gprsize] ; t63-n [idct64] psubd m%4, m%1, m%3 ; out63-n paddd m%1, m%3 ; out0+n psubd m%3, m%5, m%2 ; out32+n paddd m%2, m%5 ; out31-n REPX {vpsravd x, m11}, m%4, m%1, m%3, m%2 %endmacro IDCT64_PASS1_PACKED_END 0, 22, 24, 10, 12, 0, 15 ; out0/1,31/30,32/33,63/62 IDCT64_PASS1_PACKED_END 7, 9, 31, 13, 12, 7, 8 ; out15/14,16/17,47/46,48/49 packssdw m0, m9 packssdw m7, m22 packssdw m24, m13 packssdw m31, m10 IDCT64_PASS1_PACKED_END 1, 21, 25, 10, 12, 1, 14 ; out3/2,28/29,35/34,60/61 IDCT64_PASS1_PACKED_END 6, 16, 30, 13, 12, 6, 9 ; out12/13,19/18,44/45,51/50 packssdw m1, m16 packssdw m6, m21 packssdw m25, m13 packssdw m30, m10 IDCT64_PASS1_PACKED_END 2, 20, 26, 10, 12, 2, 13 ; out4/5,27/26,36/37,59/58 IDCT64_PASS1_PACKED_END 5, 17, 29, 13, 12, 5, 10 ; out11/10,20/21,43/42,52/53 packssdw m2, m17 packssdw m5, m20 packssdw m26, m13 packssdw m29, m10 IDCT64_PASS1_PACKED_END 3, 19, 27, 10, 12, 3, 12 ; out7/6,24/25,39/38,56/57 IDCT64_PASS1_PACKED_END 4, 18, 28, 13, 12, 4, 11 ; out8/9,23/22,40/41,55/54 packssdw m3, m18 packssdw m4, m19 packssdw m27, m13 packssdw m28, m10 ret .main_oddhalf_packed_rect2: REPX {paddd x, m13}, m0, m1 REPX {psrad x, 12 }, m0, m1 .main_oddhalf_packed: ; m0=in1 in5, m1=in7 in3 vbroadcasti32x4 m2, [o(pd_101_501)] vbroadcasti32x4 m3, [o(pd_m700_m301)] vbroadcasti32x4 m4, [o(pd_4095_4065)] vbroadcasti32x4 m5, [o(pd_4036_4085)] pmulld m2, m0 pmulld m3, m1 pmulld m0, m4 pmulld m1, m5 REPX {paddd x, m13}, m2, m3, m0, m1 REPX {psrad x, 12 }, m2, m3, m0, m1 ; m2=t32a t40a -> t32/33 t40/41, m3=t39a t47a -> t38/39 t46/47 ; m0=t63a t55a -> t62/63 t54/55, m1=t56a t48a -> t56/57 t48/49 ; end of step 1-2 vbroadcasti32x4 m10, [o(pd_401_1931)] vbroadcasti32x4 m11, [o(pd_4076_3612)] mova m4, m0 mova m5, m2 ITX_MULSUB_2D 4, 5, 8, 9, _, 13, 10, 11 vbroadcasti32x4 m10, [o(pd_3166_3920)] vbroadcasti32x4 m11, [o(pd_2598_1189)] mova m6, m3 mova m7, m1 ITX_MULSUB_2D 7, 6, 8, 9, _, 13, 10, 11, 2 ; m4=t33a t41a -> t41/42 t33/34, m5=t63a t54a -> t61/62 t53/54 ; m6=t38a t46a -> t37/38 t45/46, m7=t57a t49a -> t57/58 t49/50 ; and from earlier: ; m0=t63 t55 -> t60/63a t52/55a, m1=t56 t48 -> t56/59a t48/51a ; m2=t32 t40 -> t32/35a t40/43a, m3=t39 t47 -> t36/39a t44/47a ; end of step 3-4 punpcklqdq m22, m2, m4 ; t32a/33 or t35a/34 punpcklqdq m21, m3, m6 ; t36a/37 or t39a/38 punpckhqdq m18, m2, m4 ; t40a/41 or t43a/42 punpckhqdq m17, m3, m6 ; t44a/45 or t47a/46 punpckhqdq m6, m1, m7 ; t48a/49 or t51a/50 punpckhqdq m19, m0, m5 ; t52a/53 or t55a/54 punpcklqdq m8, m1, m7 ; t56a/57 or t59a/58 punpcklqdq m23, m0, m5 ; t60a/61 or t63a/62 mova m0, m22 mova m7, m21 mova m3, m18 mova m16, m17 mova m5, m6 mova m4, m19 mova m2, m8 mova m1, m23 ; m0/22/7/21,18/3/17/16,6/5/19/4,2/8/1/23: t32-63[a] ; step5 vpbroadcastd m10, [o(pd_799)] vpbroadcastd m11, [o(pd_4017)] ITX_MULSUB_2D 1, 22, 20, 9, _, 13, 10, 11 ; t35/34a, t60/61a ITX_MULSUB_2D 8, 7, 20, 9, _, 13, 10, 11, 2 ; t59/58a, t36/37a vpbroadcastd m10, [o(pd_3406)] vpbroadcastd m11, [o(pd_2276)] ITX_MULSUB_2D 19, 3, 20, 9, _, 13, 10, 11 ; t43/42a, t52/53a ITX_MULSUB_2D 5, 17, 20, 9, _, 13, 10, 11, 2 ; t51/50a, t44/45a ; m0-1/7/21: t32-39[a], m18-19/17-16: t40-47[a] ; m6-5/3-4: t48-55[a], m2/8/22-23: t56-63[a] ; step6 psubd m20, m0, m21 ; t39/38a paddd m0, m21 ; t32/33a psubd m21, m1, m7 ; t36a/37 paddd m1, m7 ; t35a/34 REPX {pmaxsd x, m14}, m20, m0, m21, m1 psubd m7, m16, m18 ; t40/41a paddd m16, m18 ; t47/46a REPX {pminsd x, m15}, m20, m0, m21, m1 psubd m18, m17, m19 ; t43a/42 paddd m17, m19 ; t44a/45 REPX {pmaxsd x, m14}, m7, m16, m18, m17 psubd m19, m6, m4 ; t55/54a paddd m6, m4 ; t48/49a REPX {pminsd x, m15}, m7, m16, m18, m17 psubd m4, m5, m3 ; t52a/53 paddd m5, m3 ; t51a/50 REPX {pmaxsd x, m14}, m19, m6, m4, m5 psubd m3, m23, m2 ; t56/57a paddd m23, m2 ; t63/62a REPX {pminsd x, m15}, m19, m6, m4, m5 psubd m2, m22, m8 ; t59a/58 paddd m22, m8 ; t60a/61 REPX {pmaxsd x, m14}, m3, m23, m2, m22 REPX {pminsd x, m15}, m3, m23, m2, m22 ; m0-1: t32-35[a], m17-16: t44-47[a], m6-5: t48-51[a], m22-23: t60-63[a] ; m21-20: t36-39[a], m7/18: t40-43[a], m4/19: t52-55[a], m3-2: t56-59[a] ; step7 vpbroadcastd m10, [o(pd_1567)] vpbroadcastd m11, [o(pd_3784)] ITX_MULSUB_2D 2, 21, 8, 9, _, 13, 10, 11 ; t36/37a, t59/58a ITX_MULSUB_2D 3, 20, 8, 9, _, 13, 10, 11 ; t39a/38, t56a/57 ITX_MULSUB_2D 19, 7, 8, 9, _, 13, 10, 11, 2 ; t55a/54, t40a/41 ITX_MULSUB_2D 4, 18, 8, 9, _, 13, 10, 11, 2 ; t52/53a, t43/42a ; m0-3: t32-39[a], m7,18-16: t40-47[a], m6-4,19: t48-55[a], m20-23: t56-63[a] ; step8 psubd m8, m0, m16 ; t47a/46 paddd m0, m16 ; t32a/33 psubd m16, m1, m17 ; t44/45a paddd m1, m17 ; t35/34a REPX {pmaxsd x, m14}, m8, m0, m16, m1 psubd m17, m2, m18 ; t43a/42 paddd m2, m18 ; t36a/37 REPX {pminsd x, m15}, m8, m0, m16, m1 psubd m18, m3, m7 ; t40/41a paddd m3, m7 ; t39/38a REPX {pmaxsd x, m14}, m17, m2, m18, m3 psubd m7, m23, m6 ; t48a/49 paddd m23, m6 ; t63a/62 REPX {pminsd x, m15}, m17, m2, m18, m3 psubd m6, m22, m5 ; t51/50a paddd m22, m5 ; t60/61a REPX {pmaxsd x, m14}, m7, m23, m6, m22 psubd m5, m21, m4 ; t52a/53 paddd m21, m4 ; t59a/58 REPX {pminsd x, m15}, m7, m23, m6, m22 psubd m4, m20, m19 ; t55/54a paddd m20, m19 ; t56/57a REPX {pmaxsd x, m14}, m5, m21, m4, m20 REPX {pminsd x, m15}, m5, m21, m4, m20 ; m0-3=t32-39[a], m18-16,8: t40-47[a], m7-4=t48-55[a], m20-23=t56-63[a] ; step9 REPX {pmulld x, m12}, m4, m18, m5, m17, m6, m16, m7, m8 REPX {paddd x, m13}, m4, m5, m6, m7 paddd m19, m4, m18 ; t55a/54 psubd m4, m18 ; t40a/41 paddd m18, m5, m17 ; t52/53a psubd m5, m17 ; t43/42a paddd m17, m6, m16 ; t51a/50 psubd m6, m16 ; t44a/45 paddd m16, m7, m8 ; t48/49a psubd m7, m8 ; t47/46a REPX {psrad x, 12 }, m19, m4, m18, m5, m17, m6, m16, m7 ; m4-7=t40-47[a], m16-19=t48-55[a] ret cglobal inv_txfm_add_dct_dct_64x32_10bpc, 4, 7, 0, dst, stride, c, eob lea r5, [o_base] test eobd, eobd jz .dconly PROLOGUE 4, 8, 32, -64*32, dst, stride, c, eob %undef cmp vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] cmp eobd, 136 jl .fast add cq, 64 cmp eobd, 543 jge .full call .pass1_fast ; bottomright 16x16 zero mov r7d, 16*12 jmp .lefthalf .full: call .pass1 mov r7d, 16*28 .lefthalf: mova [cq+128* 0], m0 mova [cq+128* 1], m1 mova [cq+128* 2], m2 mova [cq+128* 3], m3 mova [cq+128* 4], m14 mova [cq+128* 5], m15 mova [cq+128* 6], m16 mova [cq+128* 7], m17 mova [cq+128* 8], m22 mova [cq+128* 9], m23 mova [cq+128*10], m24 mova [cq+128*11], m25 mova [cq+128*12], m26 mova [cq+128*13], m27 mova [cq+128*14], m28 mova [cq+128*15], m29 sub cq, 64 vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] sub rsp, 16*64 call .pass1 add rsp, 16*64 lea r5, [o_base_8bpc] call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_start mov r4, dstq pxor m12, m12 call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_end lea dstq, [r4+64] mova m0, [rsp+16*mmsize] mova m1, [rsp+17*mmsize] mova m2, [rsp+18*mmsize] mova m3, [rsp+19*mmsize] mova m4, [rsp+20*mmsize] mova m5, [rsp+21*mmsize] mova m6, [rsp+22*mmsize] mova m7, [rsp+23*mmsize] mova m16, [rsp+24*mmsize] mova m17, [rsp+25*mmsize] mova m18, [rsp+26*mmsize] mova m19, [rsp+27*mmsize] mova m20, [rsp+28*mmsize] mova m21, [rsp+29*mmsize] mova m22, [rsp+30*mmsize] mova m23, [rsp+31*mmsize] call .transpose mova [cq+128* 0+64], m0 mova [cq+128* 1+64], m1 mova [cq+128* 2+64], m2 mova [cq+128* 3+64], m3 mova [cq+128* 4+64], m14 mova [cq+128* 5+64], m15 mova [cq+128* 6+64], m16 mova [cq+128* 7+64], m17 mova [cq+128* 8+64], m22 mova [cq+128* 9+64], m23 mova [cq+128*10+64], m24 mova [cq+128*11+64], m25 mova [cq+128*12+64], m26 mova [cq+128*13+64], m27 mova [cq+128*14+64], m28 mova [cq+128*15+64], m29 mova m0, [rsp+ 0*mmsize] mova m1, [rsp+ 1*mmsize] mova m2, [rsp+ 2*mmsize] mova m3, [rsp+ 3*mmsize] mova m4, [rsp+ 4*mmsize] mova m5, [rsp+ 5*mmsize] mova m6, [rsp+ 6*mmsize] mova m7, [rsp+ 7*mmsize] mova m16, [rsp+ 8*mmsize] mova m17, [rsp+ 9*mmsize] mova m18, [rsp+10*mmsize] mova m19, [rsp+11*mmsize] mova m20, [rsp+12*mmsize] mova m21, [rsp+13*mmsize] mova m22, [rsp+14*mmsize] mova m23, [rsp+15*mmsize] call .transpose call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_start pxor m12, m12 .right_zero_loop: mova [cq+r7*8+64+128*3], m12 mova [cq+r7*8+64+128*2], m12 mova [cq+r7*8+64+128*1], m12 mova [cq+r7*8+64+128*0], m12 sub r7d, 16*4 jge .right_zero_loop mov r7d, 16*28 jmp .end .fast: ; topleft 16x16 nonzero cmp eobd, 36 jl .fast2 call .pass1_fast lea r5, [o_base_8bpc] call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_fast_start mov r4, dstq pxor m12, m12 call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_end lea dstq, [r4+64] mova m0, [rsp+16*mmsize] mova m1, [rsp+17*mmsize] mova m2, [rsp+18*mmsize] mova m3, [rsp+19*mmsize] mova m4, [rsp+20*mmsize] mova m5, [rsp+21*mmsize] mova m6, [rsp+22*mmsize] mova m7, [rsp+23*mmsize] mova m16, [rsp+24*mmsize] mova m17, [rsp+25*mmsize] mova m18, [rsp+26*mmsize] mova m19, [rsp+27*mmsize] mova m20, [rsp+28*mmsize] mova m21, [rsp+29*mmsize] mova m22, [rsp+30*mmsize] mova m23, [rsp+31*mmsize] call .transpose call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_fast_start mov r7d, 16*12 pxor m12, m12 jmp .end .fast2: ; topleft 8x8 nonzero movshdup m7, [o(permB)] mova ym0, [cq+128*1] mova ym2, [cq+128*5] mova ym3, [cq+128*3] mova ym1, [cq+128*7] vpermt2q m0, m7, m2 ; 1 5 vpermt2q m1, m7, m3 ; 7 3 REPX {pmulld x, m12}, m0, m1 call m(inv_txfm_add_dct_dct_64x16_10bpc).main_oddhalf_packed_rect2 mova [rsp+ 0*mmsize], m0 mova [rsp+ 1*mmsize], m1 mova [rsp+ 2*mmsize], m2 mova [rsp+ 3*mmsize], m3 mova [rsp+ 4*mmsize], m4 mova [rsp+ 5*mmsize], m5 mova [rsp+ 6*mmsize], m6 mova [rsp+ 7*mmsize], m7 mova [rsp+ 8*mmsize], m16 mova [rsp+ 9*mmsize], m17 mova [rsp+10*mmsize], m18 mova [rsp+11*mmsize], m19 mova [rsp+12*mmsize], m20 mova [rsp+13*mmsize], m21 mova [rsp+14*mmsize], m22 mova [rsp+15*mmsize], m23 movshdup m7, [o(permB)] pmulld ym0, ym12, [cq+128*0] pmulld ym4, ym12, [cq+128*4] mova ym16, [cq+128*2] mova ym5, [cq+128*6] REPX {paddd x, ym13}, ym0, ym4 REPX {psrad x, 12 }, ym0, ym4 vpermt2q m16, m7, m5 ; 2 6 vpermq m0, m7, m0 ; 0 0 vpermq m4, m7, m4 ; 4 4 pmulld m16, m12 paddd m16, m13 psrad m16, 12 call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast3 vpbroadcastd m11, [o(pd_1)] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_end mova [rsp+16*mmsize], m24 mova [rsp+17*mmsize], m25 mova [rsp+18*mmsize], m26 mova [rsp+19*mmsize], m27 mova [rsp+20*mmsize], m28 mova [rsp+21*mmsize], m29 mova [rsp+22*mmsize], m30 mova [rsp+23*mmsize], m31 vpbroadcastd m13, [o(pd_2048)] call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_fast2_start mov r7d, 16*4 mov r4, dstq pxor m12, m12 call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_end lea dstq, [r4+64] mova m0, [rsp+16*mmsize] mova m1, [rsp+17*mmsize] mova m2, [rsp+18*mmsize] mova m3, [rsp+19*mmsize] mova m4, [rsp+20*mmsize] mova m5, [rsp+21*mmsize] mova m6, [rsp+22*mmsize] mova m7, [rsp+23*mmsize] lea r5, [o_base] vpbroadcastd m13, [o(pd_2048)] call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_fast2_start pxor m12, m12 .end: call m(inv_txfm_add_dct_dct_32x32_10bpc).pass2_end .zero_loop: mova [cq+r7*8+128*3], m12 mova [cq+r7*8+128*2], m12 mova [cq+r7*8+128*1], m12 mova [cq+r7*8+128*0], m12 sub r7d, 16*4 jge .zero_loop RET .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 32 add r6d, 128 sar r6d, 8 imul r6d, 181 add r6d, 384 sar r6d, 9 jmp m(inv_txfm_add_dct_dct_64x16_10bpc).dconly2 .pass1_fast: lea r4, [idct64_mul_16bpc] lea r6, [rsp+4*64+gprsize] pmulld m0, m12, [cq+128* 1] pmulld m3, m12, [cq+128*15] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_fast_rect2 pmulld m0, m12, [cq+128* 7] pmulld m3, m12, [cq+128* 9] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_fast_rect2 pmulld m0, m12, [cq+128* 5] pmulld m3, m12, [cq+128*11] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_fast_rect2 pmulld m0, m12, [cq+128* 3] pmulld m3, m12, [cq+128*13] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_fast_rect2 call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part2 pmulld m0, m12, [cq+128* 0] pmulld m1, m12, [cq+128* 8] pmulld m16, m12, [cq+128* 4] pmulld m17, m12, [cq+128*12] call m(idct_8x16_internal_10bpc).main_fast2_rect2 call m(idct_16x16_internal_10bpc).main_fast2_rect2 call .pass1_load_spill call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast2_rect2 jmp .pass1_end .pass1: lea r4, [idct64_mul_16bpc] lea r6, [rsp+4*64+gprsize] pmulld m0, m12, [cq+128* 1] pmulld m1, m12, [cq+128*31] pmulld m2, m12, [cq+128*17] pmulld m3, m12, [cq+128*15] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_rect2 pmulld m0, m12, [cq+128* 7] pmulld m1, m12, [cq+128*25] pmulld m2, m12, [cq+128*23] pmulld m3, m12, [cq+128* 9] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_rect2 pmulld m0, m12, [cq+128* 5] pmulld m1, m12, [cq+128*27] pmulld m2, m12, [cq+128*21] pmulld m3, m12, [cq+128*11] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_rect2 pmulld m0, m12, [cq+128* 3] pmulld m1, m12, [cq+128*29] pmulld m2, m12, [cq+128*19] pmulld m3, m12, [cq+128*13] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_rect2 call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part2 pmulld m0, m12, [cq+128* 0] pmulld m1, m12, [cq+128* 8] pmulld m2, m12, [cq+128*16] pmulld m3, m12, [cq+128*24] pmulld m16, m12, [cq+128* 4] pmulld m17, m12, [cq+128*12] pmulld m18, m12, [cq+128*20] pmulld m19, m12, [cq+128*28] call m(idct_8x16_internal_10bpc).main_fast_rect2 call m(idct_16x16_internal_10bpc).main_fast_rect2 call .pass1_load_spill pmulld m4, m12, [cq+128*18] pmulld m5, m12, [cq+128*22] pmulld m6, m12, [cq+128*26] pmulld m7, m12, [cq+128*30] call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast_rect2 .pass1_end: vpbroadcastd m11, [o(pd_1)] lea r3, [rsp+gprsize] lea r4, [cq+8*128] call m(inv_txfm_add_dct_dct_64x16_10bpc).idct64_main_end ; transpose one half immediately, we can transpose lower half later .transpose: ; transpose m0-7,16-23 psrlq m12, [permC], 24 ; 0 2 8 10 1 3 9 11 psrlq m13, m12, 32 ; 4 6 12 14 5 7 13 15 call m(inv_txfm_add_dct_dct_32x16_10bpc).transpose_16x32 punpckhqdq m22, m0, m20 ; 1 punpcklqdq m0, m20 ; 0 punpckhqdq m24, m2, m1 ; 5 punpcklqdq m1, m2, m1 ; 4 punpcklqdq m2, m14, m18 ; 8 punpckhqdq m26, m14, m18 ; 9 punpcklqdq m14, m15, m4 ; 2 punpckhqdq m23, m15, m4 ; 3 punpckhqdq m25, m3, m21 ; 7 punpcklqdq m15, m3, m21 ; 6 punpckhqdq m28, m6, m17 ; 13 punpcklqdq m3, m6, m17 ; 12 punpckhqdq m27, m5, m16 ; 11 punpcklqdq m16, m5, m16 ; 10 punpckhqdq m29, m7, m8 ; 15 punpcklqdq m17, m7, m8 ; 14 ret .pass1_load_spill: call m(inv_txfm_add_dct_dct_32x16_10bpc).idct16_sumsub mova [cq+128* 0], m0 mova [cq+128* 1], m1 pmulld m0, m12, [cq+128* 2] pmulld m1, m12, [cq+128* 6] mova [cq+128* 2], m2 mova [cq+128* 3], m3 pmulld m2, m12, [cq+128*10] pmulld m3, m12, [cq+128*14] mova [cq+128* 4], m4 mova [cq+128* 5], m5 mova [cq+128* 6], m6 mova [cq+128* 7], m7 mova [cq+128* 8], m23 mova [cq+128* 9], m22 mova [cq+128*10], m21 mova [cq+128*11], m20 mova [cq+128*12], m19 mova [cq+128*13], m18 mova [cq+128*14], m17 mova [cq+128*15], m16 ret cglobal inv_txfm_add_dct_dct_64x64_10bpc, 4, 7, 0, dst, stride, c, eob lea r5, [o_base] test eobd, eobd jz .dconly PROLOGUE 4, 9, 32, -64*32, dst, stride, c, eob %undef cmp vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] cmp eobd, 136 jl .fast add cq, 64 cmp eobd, 543 jge .full call .pass1_fast ; bottomright 16x16 zero mov r7d, 16*12 jmp .lefthalf .full: call .pass1 mov r7d, 16*28 .lefthalf: mova [cq+128* 0], m27 mova [cq+128* 1], m14 mova [cq+128* 2], m28 mova [cq+128* 3], m15 mova [cq+128* 4], m22 mova [cq+128* 5], m23 mova [cq+128* 6], m24 mova [cq+128* 7], m25 mova [cq+128* 8], m0 mova [cq+128* 9], m26 mova [cq+128*10], m20 mova [cq+128*11], m21 mova [cq+128*12], m18 mova [cq+128*13], m16 mova [cq+128*14], m17 mova [cq+128*15], m3 sub cq, 64 vpbroadcastd m12, [o(pd_2896)] vpbroadcastd m13, [o(pd_2048)] vpbroadcastd m14, [o(clip_18b_min)] vpbroadcastd m15, [o(clip_18b_max)] sub rsp, 16*64 call .pass1 sub rsp, 24*64 call m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_start mov r8, dstq pxor m31, m31 call m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_end lea dstq, [r8+64] mova m0, [rsp+56*mmsize] mova m1, [rsp+57*mmsize] mova m2, [rsp+58*mmsize] mova m3, [rsp+59*mmsize] mova m4, [rsp+60*mmsize] mova m5, [rsp+61*mmsize] mova m6, [rsp+62*mmsize] mova m7, [rsp+63*mmsize] mova m16, [rsp+64*mmsize] mova m17, [rsp+65*mmsize] mova m18, [rsp+66*mmsize] mova m19, [rsp+67*mmsize] mova m20, [rsp+68*mmsize] mova m21, [rsp+69*mmsize] mova m22, [rsp+70*mmsize] mova m23, [rsp+71*mmsize] call .transpose mova [cq+128* 0+64], m27 mova [cq+128* 1+64], m14 mova [cq+128* 2+64], m28 mova [cq+128* 3+64], m15 mova [cq+128* 4+64], m22 mova [cq+128* 5+64], m23 mova [cq+128* 6+64], m24 mova [cq+128* 7+64], m25 mova [cq+128* 8+64], m0 mova [cq+128* 9+64], m26 mova [cq+128*10+64], m20 mova [cq+128*11+64], m21 mova [cq+128*12+64], m18 mova [cq+128*13+64], m16 mova [cq+128*14+64], m17 mova [cq+128*15+64], m3 mova m0, [rsp+40*mmsize] mova m1, [rsp+41*mmsize] mova m2, [rsp+42*mmsize] mova m3, [rsp+43*mmsize] mova m4, [rsp+44*mmsize] mova m5, [rsp+45*mmsize] mova m6, [rsp+46*mmsize] mova m7, [rsp+47*mmsize] mova m16, [rsp+48*mmsize] mova m17, [rsp+49*mmsize] mova m18, [rsp+50*mmsize] mova m19, [rsp+51*mmsize] mova m20, [rsp+52*mmsize] mova m21, [rsp+53*mmsize] mova m22, [rsp+54*mmsize] mova m23, [rsp+55*mmsize] add rsp, 32*64 call .transpose lea r5, [o_base] call m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_start .right_zero_loop: REPX {mova [cq+r7*8+64+128*x], m31}, 0, 1, 2, 3 sub r7d, 16*4 jge .right_zero_loop mov r7d, 16*28 jmp .end .fast: ; topleft 16x16 nonzero cmp eobd, 36 jl .fast2 call .pass1_fast sub rsp, 24*64 vpbroadcastd m10, [o(pd_2048)] call m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_fast_start mov r8, dstq pxor m31, m31 call m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_end lea dstq, [r8+64] mova m0, [rsp+40*mmsize] mova m1, [rsp+41*mmsize] mova m2, [rsp+42*mmsize] mova m3, [rsp+43*mmsize] mova m4, [rsp+44*mmsize] mova m5, [rsp+45*mmsize] mova m6, [rsp+46*mmsize] mova m7, [rsp+47*mmsize] mova m16, [rsp+48*mmsize] mova m17, [rsp+49*mmsize] mova m18, [rsp+50*mmsize] mova m19, [rsp+51*mmsize] mova m20, [rsp+52*mmsize] mova m21, [rsp+53*mmsize] mova m22, [rsp+54*mmsize] mova m23, [rsp+55*mmsize] add rsp, 16*64 call .transpose lea r5, [o_base] vpbroadcastd m10, [o(pd_2048)] call m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_fast_start mov r7d, 16*12 jmp .end .fast2: ; topleft 8x8 nonzero movshdup m7, [o(permB)] mova ym0, [cq+128*1] mova ym2, [cq+128*5] mova ym3, [cq+128*3] mova ym1, [cq+128*7] vpermt2q m0, m7, m2 ; 1 5 vpermt2q m1, m7, m3 ; 7 3 call m(inv_txfm_add_dct_dct_64x16_10bpc).main_oddhalf_packed mova [rsp+ 0*mmsize], m0 mova [rsp+ 1*mmsize], m1 mova [rsp+ 2*mmsize], m2 mova [rsp+ 3*mmsize], m3 mova [rsp+ 4*mmsize], m4 mova [rsp+ 5*mmsize], m5 mova [rsp+ 6*mmsize], m6 mova [rsp+ 7*mmsize], m7 mova [rsp+ 8*mmsize], m16 mova [rsp+ 9*mmsize], m17 mova [rsp+10*mmsize], m18 mova [rsp+11*mmsize], m19 mova [rsp+12*mmsize], m20 mova [rsp+13*mmsize], m21 mova [rsp+14*mmsize], m22 mova [rsp+15*mmsize], m23 movshdup m7, [o(permB)] mova ym0, [cq+128*0] mova ym4, [cq+128*4] mova ym16, [cq+128*2] mova ym5, [cq+128*6] vpermt2q m16, m7, m5 ; 2 6 vpermq m0, m7, m0 ; 0 0 vpermq m4, m7, m4 ; 4 4 call m(inv_txfm_add_dct_dct_32x8_10bpc).main_fast3 vpbroadcastd m11, [o(pd_2)] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_end sub rsp, 16*64 mova [rsp+40*mmsize], m24 mova [rsp+41*mmsize], m25 mova [rsp+42*mmsize], m26 mova [rsp+43*mmsize], m27 mova [rsp+44*mmsize], m28 mova [rsp+45*mmsize], m29 mova [rsp+46*mmsize], m30 mova [rsp+47*mmsize], m31 call .pass2_fast2_start mov r7d, 16*4 mov r8, dstq pxor m31, m31 call m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_end lea dstq, [r8+64] mova m0, [rsp+40*mmsize] mova m1, [rsp+41*mmsize] mova m2, [rsp+42*mmsize] mova m3, [rsp+43*mmsize] mova m4, [rsp+44*mmsize] mova m5, [rsp+45*mmsize] mova m6, [rsp+46*mmsize] mova m7, [rsp+47*mmsize] add rsp, 8*64 lea r5, [o_base] call .pass2_fast2_start .end: pxor m31, m31 .zero_loop: REPX {mova [cq+r7*8+128*x], m31}, 0, 1, 2, 3 sub r7d, 16*4 jge .zero_loop call m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_end add rsp, 8*64 ; FIXME adjust stack_size_padded instead? RET .pass2_fast2_start: call m(inv_txfm_add_dct_dct_32x8_10bpc).transpose_8x32 punpcklqdq m27, m0, m2 ; 0 punpckhqdq m0, m2 ; 1 punpcklqdq m22, m3, m4 ; 2 punpckhqdq m26, m3, m4 ; 3 punpcklqdq m14, m5, m7 ; 4 punpckhqdq m20, m5, m7 ; 5 punpcklqdq m23, m6, m8 ; 6 punpckhqdq m21, m6, m8 ; 7 vpbroadcastd m10, [o(pd_2048)] jmp m(inv_txfm_add_dct_dct_32x64_10bpc).pass2_fast2_start .dconly: imul r6d, [cq], 181 mov [cq], eobd or r3d, 64 jmp m(inv_txfm_add_dct_dct_64x16_10bpc).dconly1 .pass1_fast: lea r4, [idct64_mul_16bpc] lea r6, [rsp+4*64+gprsize] mova m0, [cq+128* 1] mova m3, [cq+128*15] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_fast mova m0, [cq+128* 7] mova m3, [cq+128* 9] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_fast mova m0, [cq+128* 5] mova m3, [cq+128*11] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_fast mova m0, [cq+128* 3] mova m3, [cq+128*13] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1_fast call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part2 mova m0, [cq+128* 0] mova m1, [cq+128* 8] mova m16, [cq+128* 4] mova m17, [cq+128*12] call m(idct_8x16_internal_10bpc).main_fast2 call m(idct_16x16_internal_10bpc).main_fast2 call .pass1_load_spill call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast2 jmp .pass1_end .pass1: lea r4, [idct64_mul_16bpc] lea r6, [rsp+4*64+gprsize] mova m0, [cq+128* 1] mova m1, [cq+128*31] mova m2, [cq+128*17] mova m3, [cq+128*15] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1 mova m0, [cq+128* 7] mova m1, [cq+128*25] mova m2, [cq+128*23] mova m3, [cq+128* 9] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1 mova m0, [cq+128* 5] mova m1, [cq+128*27] mova m2, [cq+128*21] mova m3, [cq+128*11] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1 mova m0, [cq+128* 3] mova m1, [cq+128*29] mova m2, [cq+128*19] mova m3, [cq+128*13] call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part1 call m(inv_txfm_add_dct_dct_64x16_10bpc).main_part2 mova m0, [cq+128* 0] mova m1, [cq+128* 8] mova m2, [cq+128*16] mova m3, [cq+128*24] mova m16, [cq+128* 4] mova m17, [cq+128*12] mova m18, [cq+128*20] mova m19, [cq+128*28] call m(idct_8x16_internal_10bpc).main_fast call m(idct_16x16_internal_10bpc).main_fast call .pass1_load_spill mova m4, [cq+128*18] mova m5, [cq+128*22] mova m6, [cq+128*26] mova m7, [cq+128*30] call m(inv_txfm_add_dct_dct_32x16_10bpc).main_fast .pass1_end: vpbroadcastd m11, [o(pd_2)] lea r3, [rsp+gprsize] lea r4, [cq+8*128] call m(inv_txfm_add_dct_dct_64x16_10bpc).idct64_main_end ; transpose one half immediately, we can transpose lower half later .transpose: ; transpose m0-7,16-23 psrlq m12, [permC], 24 ; 0 2 8 10 1 3 9 11 psrlq m13, m12, 32 ; 4 6 12 14 5 7 13 15 call m(inv_txfm_add_dct_dct_32x16_10bpc).transpose_16x32 punpcklqdq m27, m0, m20 ; 0 punpckhqdq m0, m20 ; 1 punpcklqdq m24, m5, m16 ; 10 punpckhqdq m16, m5, m16 ; 11 punpcklqdq m23, m3, m21 ; 6 punpckhqdq m21, m3, m21 ; 7 punpcklqdq m25, m7, m8 ; 14 punpckhqdq m3, m7, m8 ; 15 punpcklqdq m22, m15, m4 ; 2 punpckhqdq m26, m15, m4 ; 3 punpcklqdq m15, m6, m17 ; 12 punpckhqdq m17, m6, m17 ; 13 punpcklqdq m28, m14, m18 ; 8 punpckhqdq m18, m14, m18 ; 9 punpcklqdq m14, m2, m1 ; 4 punpckhqdq m20, m2, m1 ; 5 ret .pass1_load_spill: call m(inv_txfm_add_dct_dct_32x16_10bpc).idct16_sumsub mova [cq+128* 0], m0 mova [cq+128* 1], m1 mova m0, [cq+128* 2] mova m1, [cq+128* 6] mova [cq+128* 2], m2 mova [cq+128* 3], m3 mova m2, [cq+128*10] mova m3, [cq+128*14] mova [cq+128* 4], m4 mova [cq+128* 5], m5 mova [cq+128* 6], m6 mova [cq+128* 7], m7 mova [cq+128* 8], m23 mova [cq+128* 9], m22 mova [cq+128*10], m21 mova [cq+128*11], m20 mova [cq+128*12], m19 mova [cq+128*13], m18 mova [cq+128*14], m17 mova [cq+128*15], m16 ret %endif ; ARCH_X86_64
¤ Dauer der Verarbeitung: 0.195 Sekunden (vorverarbeitet am 2026-04-27) ¤*© Formatika GbR, Deutschland |
|
||||||||||||||
2026-05-26