History

avcodec/x86/vp9: Add AVX-512ICL for 16x16 and 32x32 10bpc inverse transforms

Henrik Gramner committed 1 year ago

Commit eda0ac71 parent df967d04 files · +1176 −2Browse files

Unified · Split

4 files
  1. libavcodec/vp9dec.h +2 −2
  2. libavcodec/x86/Makefile +1 −0
  3. libavcodec/x86/vp9dsp_init_16bpp_template.c +8 −0
  4. libavcodec/x86/vp9itxfm_16bpp_avx512.asm +1165 −0
libavcodec/vp9dec.h+2−2
@@ -220,8 +220,8 @@ struct VP9TileData { · 219 unmodified lines
220220 DECLARE_ALIGNED(8, uint8_t, left_ref_ctx)[8];
221221 DECLARE_ALIGNED(8, uint8_t, left_filter_ctx)[8];
222222 // block reconstruction intermediates
223− DECLARE_ALIGNED(32, uint8_t, tmp_y)[64 * 64 * 2];
224− DECLARE_ALIGNED(32, uint8_t, tmp_uv)[2][64 * 64 * 2];
223+ DECLARE_ALIGNED(64, uint8_t, tmp_y)[64 * 64 * 2];
224+ DECLARE_ALIGNED(64, uint8_t, tmp_uv)[2][64 * 64 * 2];
225225 struct { int x, y; } min_mv, max_mv;
226226 int16_t *block_base, *block, *uvblock_base[2], *uvblock[2];
227227 uint8_t *eob_base, *uveob_base[2], *eob, *uveob[2];
libavcodec/x86/Makefile+1−0
@@ -186,6 +186,7 @@ X86ASM-OBJS-$(CONFIG_VP9_DECODER) += x86/vp9intrapred.o \ · 185 unmodified lines
186186 x86/vp9itxfm.o \
187187 x86/vp9itxfm_avx512.o \
188188 x86/vp9itxfm_16bpp.o \
189+ x86/vp9itxfm_16bpp_avx512.o \
189190 x86/vp9lpf.o \
190191 x86/vp9lpf_16bpp.o \
191192 x86/vp9mc.o \
libavcodec/x86/vp9dsp_init_16bpp_template.c+8−0
@@ -127,6 +127,8 @@ decl_itxfm_func(iwht, iwht, 4, BPC, mmxext); · 126 unmodified lines
127127 #if BPC == 10
128128 decl_itxfm_func(idct, idct, 4, BPC, mmxext);
129129 decl_itxfm_funcs(4, BPC, ssse3);
130+decl_itxfm_funcs(16, BPC, avx512icl);
131+decl_itxfm_func(idct, idct, 32, BPC, avx512icl);
130132 #else
131133 decl_itxfm_func(idct, idct, 4, BPC, sse2);
132134 #endif
@@ -233,6 +235,12 @@ av_cold void INIT_FUNC(VP9DSPContext *dsp, int bitexact) · 100 unmodified lines
233235 #endif
234236 }
235237
238+#if ARCH_X86_64 && BPC == 10
239+ if (EXTERNAL_AVX512ICL(cpu_flags)) {
240+ init_itx_funcs(TX_16X16, 16, BPC, avx512icl);
241+ init_itx_func_one(TX_32X32, idct, idct, 32, BPC, avx512icl);
242+ }
243+#endif
236244 #endif /* HAVE_X86ASM */
237245
238246 ff_vp9dsp_init_16bpp_x86(dsp);
libavcodec/x86/vp9itxfm_16bpp_avx512.asm+1165−0
@@ -0,0 +1,1165 @@
1+;******************************************************************************
2+;* VP9 IDCT SIMD optimizations
3+;*
4+;* Copyright (C) 2025 Two Orioles, LLC
5+;*
6+;* This file is part of FFmpeg.
7+;*
8+;* FFmpeg is free software; you can redistribute it and/or
9+;* modify it under the terms of the GNU Lesser General Public
10+;* License as published by the Free Software Foundation; either
11+;* version 2.1 of the License, or (at your option) any later version.
12+;*
13+;* FFmpeg is distributed in the hope that it will be useful,
14+;* but WITHOUT ANY WARRANTY; without even the implied warranty of
15+;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16+;* Lesser General Public License for more details.
17+;*
18+;* You should have received a copy of the GNU Lesser General Public
19+;* License along with FFmpeg; if not, write to the Free Software
20+;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21+;******************************************************************************
22+
23+%include "libavutil/x86/x86util.asm"
24+
25+%if ARCH_X86_64 && HAVE_AVX512ICL_EXTERNAL
26+
27+SECTION_RODATA 64
28+
29+; Thw following set of constants are ordered to form the
30+; qword shuffle mask { 0, 2, 4, 6, 1, 3, 5, 7 }
31+%define deintq_perm pd_5520
32+pd_5520: dd 5520
33+pd_9760: dd 9760
34+pd_10394: dd 10394
35+pd_15426: dd 15426
36+pd_804: dd 804
37+pd_2404: dd 2404
38+pd_6270: dd 6270
39+pd_9102: dd 9102
40+pd_11585: dd 11585
41+pd_12665: dd 12665
42+pd_7723: dd 7723
43+pd_14811: dd 14811
44+pd_7005: dd 7005
45+pd_14053: dd 14053
46+pd_8423: dd 8423
47+pd_13623: dd 13623
48+
49+pixel_clip: times 2 dw 0x7c00
50+pixel_clip6: dd 2031648 ; 32 + (pixel_clip << 6)
51+pd_532480: dd 532480 ; 8192 + (32 << 14)
52+pd_8192: dd 8192
53+
54+pd_1606: dd 1606
55+pd_3196: dd 3196
56+pd_3981: dd 3981
57+pd_4756: dd 4756
58+pd_11003: dd 11003
59+pd_12140: dd 12140
60+pd_13160: dd 13160
61+pd_14449: dd 14449
62+pd_15137: dd 15137
63+pd_15679: dd 15679
64+pd_15893: dd 15893
65+pd_16069: dd 16069
66+pd_16207: dd 16207
67+pd_16305: dd 16305
68+pd_16364: dd 16364
69+
70+SECTION .text
71+
72+%define o_base (deintq_perm+128)
73+%define o(x) (r5 - o_base + (x))
74+%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)
75+
76+; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12
77+; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12
78+; skip round/shift if rnd is not a number
79+%macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], inv_dst2
80+%if %8 < 32
81+ pmulld m%4, m%1, m%8
82+ pmulld m%3, m%2, m%8
83+%else
84+ vpbroadcastd m%3, [o(pd_%8)]
85+ pmulld m%4, m%1, m%3
86+ pmulld m%3, m%2
87+%endif
88+%if %7 < 32
89+ pmulld m%1, m%7
90+ pmulld m%2, m%7
91+%else
92+ vpbroadcastd m%5, [o(pd_%7)]
93+ pmulld m%1, m%5
94+ pmulld m%2, m%5
95+%endif
96+%if %9
97+ psubd m%4, m%6, m%4
98+ psubd m%2, m%4, m%2
99+%else
100+%ifnum %6
101+ paddd m%4, m%6
102+%endif
103+ paddd m%2, m%4
104+%endif
105+%ifnum %6
106+ paddd m%1, m%6
107+%endif
108+ psubd m%1, m%3
109+%ifnum %6
110+ psrad m%2, 14
111+ psrad m%1, 14
112+%endif
113+%endmacro
114+
115+%macro WRAP_YMM 1+
116+ INIT_YMM cpuname
117+ %1
118+ INIT_ZMM cpuname
119+%endmacro
120+
121+%macro TRANSPOSE_4D 5 ; in[1-4], tmp
122+ punpckhdq m%5, m%3, m%4 ; c2 d2 c3 d3
123+ punpckldq m%3, m%4 ; c0 d0 c1 d1
124+ punpckhdq m%4, m%1, m%2 ; a2 b2 a3 b3
125+ punpckldq m%1, m%2 ; a0 b0 a1 b1
126+ punpckhqdq m%2, m%1, m%3 ; a1 b1 c1 d1
127+ punpcklqdq m%1, m%3 ; a0 b0 c0 d0
128+ punpcklqdq m%3, m%4, m%5 ; a2 b2 c2 d2
129+ punpckhqdq m%4, m%5 ; a3 b3 c3 d3
130+%endmacro
131+
132+%macro TRANSPOSE_4DQ 5 ; in[1-4], tmp
133+ vshufi32x4 m%5, m%3, m%4, q3232 ; c2 c3 d2 d3
134+ vinserti32x8 m%3, ym%4, 1 ; c0 c1 d0 d1
135+ vshufi32x4 m%4, m%1, m%2, q3232 ; a2 a3 b2 b3
136+ vinserti32x8 m%1, ym%2, 1 ; a0 a1 b0 b1
137+ vshufi32x4 m%2, m%1, m%3, q3131 ; a1 b1 c1 d1
138+ vshufi32x4 m%1, m%3, q2020 ; a0 b0 c0 d0
139+ vshufi32x4 m%3, m%4, m%5, q2020 ; a2 b2 c2 d2
140+ vshufi32x4 m%4, m%5, q3131 ; a3 b3 c3 d3
141+%endmacro
142+
143+%macro INV_TXFM_FN 3-4 0 ; type1, type2, size, eob_offset
144+cglobal vp9_i%1_i%2_%3_add_10, 4, 5, 0, dst, stride, c, eob, tx2
145+ %define %%p1 m(vp9_i%1_%3_internal_10)
146+ lea r5, [o_base]
147+ ; Jump to the 1st txfm function if we're not taking the fast path, which
148+ ; in turn performs an indirect jump to the 2nd txfm function.
149+ lea tx2q, [m(vp9_i%2_%3_internal_10).pass2]
150+%ifidn %1_%2, dct_dct
151+ dec eobd
152+ jnz %%p1
153+%else
154+%if %4
155+ add eobd, %4
156+%endif
157+ ; jump to the 1st txfm function unless it's located directly after this
158+ times ((%%end - %%p1) >> 31) & 1 jmp %%p1
159+ALIGN function_align
160+%%end:
161+%endif
162+%endmacro
163+
164+%macro INV_TXFM_16X16_FN 2-3 0 ; type1, type2, eob_offset
165+ INV_TXFM_FN %1, %2, 16x16, %3
166+%ifidn %1_%2, dct_dct
167+ imul r6d, [cq], 11585
168+ vpbroadcastd ym3, [o(pixel_clip)]
169+ mov [cq], r3d
170+ add r6d, 8192
171+ sar r6d, 14
172+ imul r6d, 11585
173+ or r3d, 8
174+ add r6d, 532480
175+ sar r6d, 20
176+ vpbroadcastw ym2, r6d
177+ paddsw ym2, ym3
178+.dconly_loop:
179+ paddsw ym0, ym2, [dstq+strideq*0]
180+ paddsw ym1, ym2, [dstq+strideq*1]
181+ psubusw ym0, ym3
182+ psubusw ym1, ym3
183+ mova [dstq+strideq*0], ym0
184+ mova [dstq+strideq*1], ym1
185+ lea dstq, [dstq+strideq*2]
186+ dec r3d
187+ jg .dconly_loop
188+ RET
189+%endif
190+%endmacro
191+
192+%macro IDCT16_PART1 0
193+%if mmsize == 64
194+.main_part1_fast:
195+%endif
196+ pmulld m15, m1, [o(pd_16305)] {bcstd} ; t15a
197+ pmulld m1, [o(pd_1606)] {bcstd} ; t8a
198+ pmulld m9, m7, [o(pd_10394)] {bcstd} ; t9a
199+ pmulld m7, [o(pd_12665)] {bcstd} ; t14a
200+ pmulld m11, m5, [o(pd_14449)] {bcstd} ; t13a
201+ pmulld m5, [o(pd_7723)] {bcstd} ; t10a
202+ pmulld m13, m3, [o(pd_4756)] {bcstd} ; t11a
203+ pmulld m3, [o(pd_15679)] {bcstd} ; t12a
204+ pmulld m10, m6, [o(pd_9102)] {bcstd} ; t5a
205+ pmulld m6, [o(pd_13623)] {bcstd} ; t6a
206+ pmulld m14, m2, [o(pd_16069)] {bcstd} ; t7a
207+ pmulld m2, [o(pd_3196)] {bcstd} ; t4a
208+ pmulld m12, m4, [o(pd_15137)] {bcstd} ; t3
209+ pmulld m4, [o(pd_6270)] {bcstd} ; t2
210+ pmulld m0, m21
211+ REPX {psubd x, m20, x}, m9, m13, m10
212+ paddd m0, m20
213+ mova m18, m0
214+%if mmsize == 64 ; for the ymm variant we only ever use the fast path
215+ jmp %%main_part1b
216+.main_part1:
217+ ITX_MULSUB_2D 1, 15, 16, 17, 18, _, 1606, 16305 ; t8a, t15a
218+ ITX_MULSUB_2D 9, 7, 16, 17, 18, _, 12665, 10394 ; t9a, t14a
219+ ITX_MULSUB_2D 5, 11, 16, 17, 18, _, 7723, 14449 ; t10a, t13a
220+ ITX_MULSUB_2D 13, 3, 16, 17, 18, _, 15679, 4756 ; t11a, t12a
221+ ITX_MULSUB_2D 10, 6, 16, 17, 18, _, 13623, 9102 ; t5a, t6a
222+ ITX_MULSUB_2D 2, 14, 16, 17, 18, _, 3196, 16069 ; t4a, t7a
223+ ITX_MULSUB_2D 4, 12, 16, 17, 18, _, 6270, 15137 ; t2, t3
224+ pmulld m0, m21
225+ pmulld m8, m21
226+ REPX {paddd x, m20}, m0, m9, m13, m10
227+ psubd m18, m0, m8 ; t1
228+ paddd m0, m8 ; t0
229+%%main_part1b:
230+%endif
231+ vpbroadcastd m19, [o(pd_15137)]
232+ vpbroadcastd m16, [o(pd_6270)]
233+ REPX {paddd x, m20}, m15, m7, m1, m11, m3, m5
234+ REPX {psrad x, 14 }, m15, m7, m1, m9, m11, m3, m5, m13
235+ paddd m17, m15, m7 ; t15
236+ psubd m15, m7 ; t14
237+ psubd m7, m3, m11 ; t13
238+ paddd m3, m11 ; t12
239+ psubd m11, m13, m5 ; t10
240+ paddd m5, m13 ; t11
241+ psubd m13, m1, m9 ; t9
242+ paddd m1, m9 ; t8
243+ ITX_MULSUB_2D 15, 13, 8, 9, _, 20, 16, 19 ; t9a, t14a
244+ ITX_MULSUB_2D 7, 11, 8, 9, _, 20, 16, 19, 2 ; t13a, t10a
245+ paddd m16, m1, m5 ; t8a
246+ psubd m1, m5 ; t11a
247+ paddd m8, m15, m11 ; t9
248+ psubd m15, m11 ; t10
249+ psubd m11, m17, m3 ; t12a
250+ paddd m17, m3 ; t15a
251+ psubd m9, m13, m7 ; t13
252+ paddd m13, m7 ; t14
253+ REPX {pmulld x, m21}, m11, m9, m1, m15
254+ REPX {paddd x, m20}, m2, m6, m14
255+ REPX {psrad x, 14 }, m10, m2, m6, m14
256+ psubd m3, m2, m10 ; t5a
257+ paddd m10, m2 ; t4
258+ paddd m11, m20
259+ psubd m5, m11, m1 ; t11
260+ paddd m11, m1 ; t12
261+ psubd m1, m14, m6 ; t6a
262+ paddd m14, m6 ; t7
263+ pmulld m1, m21
264+ pmulld m3, m21
265+ paddd m4, m20
266+ paddd m12, m20
267+ REPX {psrad x, 14 }, m4, m12, m0, m18
268+ paddd m9, m20
269+ paddd m2, m9, m15 ; t13a
270+ psubd m9, m15 ; t10a
271+ paddd m1, m20
272+ psubd m6, m1, m3 ; t5
273+ paddd m1, m3 ; t6
274+ REPX {psrad x, 14}, m6, m1, m11, m5, m2, m9
275+%endmacro
276+
277+%macro IDCT16_PART2 0
278+ psubd m3, m0, m12 ; t3
279+ paddd m0, m12 ; t0
280+ psubd m12, m18, m4 ; t2
281+ paddd m18, m4 ; t1
282+ psubd m4, m3, m10 ; t4
283+ paddd m3, m10 ; t3
284+ psubd m10, m12, m6 ; t5
285+ paddd m12, m6 ; t2
286+ psubd m6, m18, m1 ; t6
287+ paddd m1, m18 ; t1
288+ psubd m7, m0, m14 ; t7
289+ paddd m0, m14 ; t0
290+ psubd m15, m0, m17 ; out15
291+ paddd m0, m17 ; out0
292+ psubd m14, m1, m13 ; out14
293+ paddd m1, m13 ; out1
294+ psubd m13, m12, m2 ; out13
295+ paddd m2, m12 ; out2
296+ psubd m12, m3, m11 ; out12
297+ paddd m3, m11 ; out3
298+ psubd m11, m4, m5 ; out11
299+ paddd m4, m5 ; out4
300+ paddd m5, m10, m9 ; out5
301+ psubd m10, m9 ; out10
302+ psubd m9, m6, m8 ; out9
303+ paddd m6, m8 ; out6
304+ psubd m8, m7, m16 ; out8
305+ paddd m7, m16 ; out7
306+%endmacro
307+
308+INIT_ZMM avx512icl
309+INV_TXFM_16X16_FN dct, dct
310+INV_TXFM_16X16_FN dct, adst, 39-23-1
311+
312+cglobal vp9_idct_16x16_internal_10, 0, 7, 22, dst, stride, c, eob, tx2
313+ mova m0, [cq+64* 0]
314+ mova m1, [cq+64* 1]
315+ mova m2, [cq+64* 2]
316+ mova m3, [cq+64* 3]
317+ mova m4, [cq+64* 4]
318+ mova m5, [cq+64* 5]
319+ mova m6, [cq+64* 6]
320+ mova m7, [cq+64* 7]
321+ vpbroadcastd m20, [o(pd_8192)]
322+ vpbroadcastd m21, [o(pd_11585)]
323+ sub eobd, 38
324+ jl .pass1_fast
325+ mova m8, [cq+64* 8]
326+ mova m9, [cq+64* 9]
327+ mova m10, [cq+64*10]
328+ mova m11, [cq+64*11]
329+ mova m12, [cq+64*12]
330+ mova m13, [cq+64*13]
331+ mova m14, [cq+64*14]
332+ mova m15, [cq+64*15]
333+ call .main_part1
334+ call .main_part2
335+.pass1_end:
336+ TRANSPOSE_4DQ 0, 4, 8, 12, 16
337+ TRANSPOSE_4DQ 1, 5, 9, 13, 16
338+ TRANSPOSE_4DQ 2, 6, 10, 14, 16
339+ TRANSPOSE_4DQ 3, 7, 11, 15, 16
340+ TRANSPOSE_4D 8, 9, 10, 11, 16
341+ TRANSPOSE_4D 12, 13, 14, 15, 16
342+ mov r6d, 64*12
343+ jmp .pass1_transpose_end
344+.pass1_fast:
345+ WRAP_YMM IDCT16_PART1
346+ WRAP_YMM IDCT16_PART2
347+.pass1_fast_end:
348+ vinserti32x8 m0, ym4, 1
349+ vinserti32x8 m8, ym12, 1
350+ vinserti32x8 m1, ym5, 1
351+ vinserti32x8 m9, ym13, 1
352+ vinserti32x8 m2, ym6, 1
353+ vinserti32x8 m10, ym14, 1
354+ vinserti32x8 m3, ym7, 1
355+ vinserti32x8 m11, ym15, 1
356+ vshufi32x4 m4, m0, m8, q3131
357+ vshufi32x4 m0, m8, q2020
358+ vshufi32x4 m5, m1, m9, q3131
359+ vshufi32x4 m1, m9, q2020
360+ vshufi32x4 m6, m2, m10, q3131
361+ vshufi32x4 m2, m10, q2020
362+ vshufi32x4 m7, m3, m11, q3131
363+ vshufi32x4 m3, m11, q2020
364+ mov r6d, 64*4
365+.pass1_transpose_end:
366+ pxor m16, m16
367+.zero_loop:
368+ mova [cq+r6+64*0], m16
369+ mova [cq+r6+64*1], m16
370+ mova [cq+r6+64*2], m16
371+ mova [cq+r6+64*3], m16
372+ sub r6d, 64*4
373+ jge .zero_loop
374+ TRANSPOSE_4D 0, 1, 2, 3, 16
375+ TRANSPOSE_4D 4, 5, 6, 7, 16
376+ jmp tx2q
377+.pass2:
378+ test eobd, eobd
379+ jl .pass2_fast
380+ call .main_part1
381+ jmp .pass2_end
382+.pass2_fast:
383+ call .main_part1_fast
384+.pass2_end:
385+ vpbroadcastd m3, [o(pixel_clip6)]
386+ paddd m0, m3
387+ paddd m18, m3
388+ call .main_part2
389+ REPX {psrad x, 6}, m0, m1, m2, m3
390+ packssdw m0, m1
391+ lea r6, [strideq*3]
392+ packssdw m1, m2, m3
393+ mova m2, [o(deintq_perm)]
394+ vpbroadcastd m3, [o(pixel_clip)]
395+ REPX {psrad x, 6}, m4, m5, m6, m7
396+ call .write_16x4
397+ packssdw m0, m4, m5
398+ packssdw m1, m6, m7
399+ REPX {psrad x, 6}, m8, m9, m10, m11
400+ call .write_16x4
401+ packssdw m0, m8, m9
402+ packssdw m1, m10, m11
403+.pass2_end2:
404+ REPX {psrad x, 6}, m12, m13, m14, m15
405+ call .write_16x4
406+ packssdw m0, m12, m13
407+ packssdw m1, m14, m15
408+ call .write_16x4
409+ RET
410+ALIGN function_align
411+.write_16x4:
412+ mova ym16, [dstq+strideq*0]
413+ vinserti32x8 m16, [dstq+strideq*1], 1
414+ mova ym17, [dstq+strideq*2]
415+ vinserti32x8 m17, [dstq+r6 ], 1
416+ vpermq m0, m2, m0
417+ vpermq m1, m2, m1
418+ paddsw m16, m0
419+ paddsw m17, m1
420+ psubusw m16, m3
421+ psubusw m17, m3
422+ mova [dstq+strideq*0], ym16
423+ vextracti32x8 [dstq+strideq*1], m16, 1
424+ mova [dstq+strideq*2], ym17
425+ vextracti32x8 [dstq+r6 ], m17, 1
426+ lea dstq, [dstq+strideq*4]
427+ ret
428+ALIGN function_align
429+ IDCT16_PART1
430+ ret
431+ALIGN function_align
432+.main_part2:
433+ IDCT16_PART2
434+ ret
435+
436+%macro IADST16_PART1 0
437+%if mmsize == 64
438+.main_part1_fast:
439+%endif
440+ pmulld m15, m0, [o(pd_16364)] {bcstd} ; t1
441+ pmulld m0, [o(pd_804)] {bcstd} ; t0
442+ pmulld m13, m2, [o(pd_15893)] {bcstd} ; t3
443+ pmulld m2, [o(pd_3981)] {bcstd} ; t2
444+ pmulld m11, m4, [o(pd_14811)] {bcstd} ; t5
445+ pmulld m4, [o(pd_7005)] {bcstd} ; t4
446+ pmulld m9, m6, [o(pd_13160)] {bcstd} ; t7
447+ pmulld m6, [o(pd_9760)] {bcstd} ; t6
448+ pmulld m8, m7, [o(pd_11003)] {bcstd} ; t8
449+ pmulld m7, [o(pd_12140)] {bcstd} ; t9
450+ pmulld m10, m5, [o(pd_8423)] {bcstd} ; t10
451+ pmulld m5, [o(pd_14053)] {bcstd} ; t11
452+ pmulld m12, m3, [o(pd_5520)] {bcstd} ; t12
453+ pmulld m3, [o(pd_15426)] {bcstd} ; t13
454+ pmulld m14, m1, [o(pd_2404)] {bcstd} ; t14
455+ pmulld m1, [o(pd_16207)] {bcstd} ; t15
456+ REPX {psubd x, m20, x}, m15, m13, m11, m9
457+%if mmsize == 64 ; for the ymm variant we only ever use the fast path
458+ jmp %%main_part1b
459+ALIGN function_align
460+.main_part1:
461+ ITX_MULSUB_2D 15, 0, 16, 17, 18, _, 804, 16364 ; t1, t0
462+ ITX_MULSUB_2D 13, 2, 16, 17, 18, _, 3981, 15893 ; t3, t2
463+ ITX_MULSUB_2D 11, 4, 16, 17, 18, _, 7005, 14811 ; t5, t4
464+ ITX_MULSUB_2D 9, 6, 16, 17, 18, _, 9760, 13160 ; t7, t6
465+ ITX_MULSUB_2D 7, 8, 16, 17, 18, _, 12140, 11003 ; t9, t8
466+ ITX_MULSUB_2D 5, 10, 16, 17, 18, _, 14053, 8423 ; t11, t10
467+ ITX_MULSUB_2D 3, 12, 16, 17, 18, _, 15426, 5520 ; t13, t12
468+ ITX_MULSUB_2D 1, 14, 16, 17, 18, _, 16207, 2404 ; t15, t14
469+ REPX {paddd x, m20}, m15, m13, m11, m9
470+%%main_part1b:
471+%endif
472+ REPX {paddd x, m20}, m0, m2, m4, m6
473+ psubd m16, m2, m10 ; t10a
474+ paddd m2, m10 ; t2a
475+ psubd m10, m9, m1 ; t15a
476+ paddd m9, m1 ; t7a
477+ psubd m1, m13, m5 ; t11a
478+ paddd m13, m5 ; t3a
479+ psubd m5, m6, m14 ; t14a
480+ paddd m6, m14 ; t6a
481+ REPX {psrad x, 14}, m16, m10, m1, m5
482+ psubd m14, m0, m8 ; t8a
483+ paddd m0, m8 ; t0a
484+ psubd m8, m15, m7 ; t9a
485+ paddd m15, m7 ; t1a
486+ psubd m7, m4, m12 ; t12a
487+ paddd m4, m12 ; t4a
488+ paddd m12, m11, m3 ; t5a
489+ psubd m11, m3 ; t13a
490+ REPX {psrad x, 14}, m14, m8, m7, m11
491+ vpbroadcastd m19, [o(pd_9102)]
492+ vpbroadcastd m18, [o(pd_13623)]
493+ ITX_MULSUB_2D 16, 1, 3, 17, _, _, 18, 19 ; t11, t10
494+ ITX_MULSUB_2D 10, 5, 3, 17, _, _, 19, 18 ; t14, t15
495+ vpbroadcastd m19, [o(pd_16069)]
496+ vpbroadcastd m18, [o(pd_3196)]
497+ ITX_MULSUB_2D 14, 8, 3, 17, _, _, 18, 19 ; t9, t8
498+ ITX_MULSUB_2D 11, 7, 3, 17, _, _, 19, 18 ; t12, t13
499+ vpbroadcastd m19, [o(pd_6270)]
500+ vpbroadcastd m18, [o(pd_15137)]
501+ REPX {psrad x, 14}, m15, m12, m0, m4
502+ psubd m3, m15, m12 ; t5
503+ paddd m15, m12 ; t1
504+ psubd m12, m0, m4 ; t4
505+ paddd m0, m4 ; t0
506+ REPX {psrad x, 14}, m2, m6, m13, m9
507+ psubd m4, m2, m6 ; t6
508+ paddd m2, m6 ; t2
509+ psubd m6, m13, m9 ; t7
510+ paddd m9, m13 ; t3
511+ REPX {paddd x, m20}, m8, m14, m1, m16
512+ psubd m13, m8, m11 ; t12a
513+ paddd m8, m11 ; t8a
514+ psubd m11, m14, m7 ; t13a
515+ paddd m14, m7 ; t9a
516+ psubd m7, m1, m10 ; t14a
517+ paddd m1, m10 ; t10a
518+ psubd m10, m16, m5 ; t15a
519+ paddd m16, m5 ; t11a
520+ REPX {psrad x, 14}, m13, m11, m7, m10
521+ ITX_MULSUB_2D 12, 3, 5, 17, _, _, 19, 18 ; t5a, t4a
522+ ITX_MULSUB_2D 6, 4, 5, 17, _, _, 18, 19 ; t6a, t7a
523+ ITX_MULSUB_2D 13, 11, 5, 17, _, _, 19, 18 ; t13, t12
524+ ITX_MULSUB_2D 10, 7, 5, 17, _, _, 18, 19 ; t14, t15
525+ REPX {psrad x, 14}, m8, m1, m14, m16
526+ psubd m5, m8, m1 ; t10
527+ paddd m1, m8 ; -out1
528+ psubd m8, m15, m9 ; t3a
529+ paddd m15, m9 ; -out15
530+ psubd m9, m14, m16 ; t11
531+ paddd m14, m16 ; out14
532+ psubd m16, m0, m2 ; t2a
533+ paddd m0, m2 ; out0
534+ REPX {paddd x, m20}, m11, m13, m12, m3
535+ paddd m2, m11, m10 ; out2
536+ psubd m11, m10 ; t14a
537+ psubd m10, m13, m7 ; t15a
538+ paddd m13, m7 ; -out13
539+ psubd m7, m12, m4 ; t7
540+ paddd m12, m4 ; out12
541+ psubd m4, m3, m6 ; t6
542+ paddd m3, m6 ; -out3
543+ REPX {psrad x, 14}, m10, m7, m11, m4
544+ REPX {pmulld x, m21}, m9, m10, m7, m8, m5, m11, m4, m16
545+ REPX {psrad x, 14}, m2, m13, m12, m3
546+%endmacro
547+
548+%macro IADST16_PART2 0
549+ paddd m9, m20
550+ psubd m10, m20, m10
551+ paddd m7, m20
552+ psubd m8, m20, m8
553+ paddd m6, m9, m5 ; out6
554+ psubd m9, m5 ; out9
555+ psubd m5, m10, m11 ; out5
556+ paddd m10, m11 ; out10
557+ psubd m11, m7, m4 ; out11
558+ paddd m4, m7 ; out4
559+ psubd m7, m8, m16 ; out7
560+ paddd m8, m16 ; out8
561+%endmacro
562+
563+%macro IADST16_PASS1_END 0
564+ pxor m16, m16
565+ psubd m1, m16, m1
566+ psubd m3, m16, m3
567+ psubd m13, m16, m13
568+ psubd m15, m16, m15
569+ REPX {psrad x, 14}, m4, m5, m6, m7, m8, m9, m10, m11
570+%endmacro
571+
572+INV_TXFM_16X16_FN adst, dct, 39-18
573+INV_TXFM_16X16_FN adst, adst
574+
575+cglobal vp9_iadst_16x16_internal_10, 0, 7, 22, dst, stride, c, eob, tx2
576+ mova m0, [cq+64* 0]
577+ mova m1, [cq+64* 1]
578+ mova m2, [cq+64* 2]
579+ mova m3, [cq+64* 3]
580+ mova m4, [cq+64* 4]
581+ mova m5, [cq+64* 5]
582+ mova m6, [cq+64* 6]
583+ mova m7, [cq+64* 7]
584+ vpbroadcastd m20, [o(pd_8192)]
585+ vpbroadcastd m21, [o(pd_11585)]
586+ sub eobd, 39
587+ jl .pass1_fast
588+ mova m8, [cq+64* 8]
589+ mova m9, [cq+64* 9]
590+ mova m10, [cq+64*10]
591+ mova m11, [cq+64*11]
592+ mova m12, [cq+64*12]
593+ mova m13, [cq+64*13]
594+ mova m14, [cq+64*14]
595+ mova m15, [cq+64*15]
596+ call .main_part1
597+ call .main_part2
598+ IADST16_PASS1_END
599+ jmp m(vp9_idct_16x16_internal_10).pass1_end
600+.pass1_fast:
601+ WRAP_YMM IADST16_PART1
602+ WRAP_YMM IADST16_PART2
603+ WRAP_YMM IADST16_PASS1_END
604+ jmp m(vp9_idct_16x16_internal_10).pass1_fast_end
605+.pass2:
606+ test eobd, eobd
607+ jl .pass2_fast
608+ call .main_part1
609+ jmp .pass2_end
610+.pass2_fast:
611+ call .main_part1_fast
612+.pass2_end:
613+ vpbroadcastd m20, [o(pd_532480)]
614+ call .main_part2
615+ vpbroadcastd m16, [o(pixel_clip6)]
616+ REPX {paddd x, m16}, m0, m2, m12, m14
617+ REPX {psubd x, m16, x}, m1, m3, m13, m15
618+ REPX {psrad x, 6}, m0, m1, m2, m3
619+ packssdw m0, m1
620+ lea r6, [strideq*3]
621+ packssdw m1, m2, m3
622+ mova m2, [o(deintq_perm)]
623+ vpbroadcastd m3, [o(pixel_clip)]
624+ REPX {psrad x, 20}, m4, m5, m6, m7
625+ call m(vp9_idct_16x16_internal_10).write_16x4
626+ packssdw m0, m4, m5
627+ packssdw m1, m6, m7
628+ paddsw m0, m3
629+ paddsw m1, m3
630+ REPX {psrad x, 20}, m8, m9, m10, m11
631+ call m(vp9_idct_16x16_internal_10).write_16x4
632+ packssdw m0, m8, m9
633+ packssdw m1, m10, m11
634+ paddsw m0, m3
635+ paddsw m1, m3
636+ jmp m(vp9_idct_16x16_internal_10).pass2_end2
637+ALIGN function_align
638+ IADST16_PART1
639+ ret
640+ALIGN function_align
641+.main_part2:
642+ IADST16_PART2
643+ ret
644+
645+cglobal vp9_idct_idct_32x32_add_10, 4, 7, 23, 64*64, dst, stride, c, eob
646+%undef cmp
647+ lea r5, [o_base]
648+ dec eobd
649+ jnz .pass1
650+ imul r6d, [cq], 11585
651+ vpbroadcastd m3, [o(pixel_clip)]
652+ mov [cq], r3d
653+ add r6d, 8192
654+ sar r6d, 14
655+ imul r6d, 11585
656+ or r3d, 16
657+ add r6d, 532480
658+ sar r6d, 20
659+ vpbroadcastw m2, r6d
660+ paddsw m2, m3
661+.dconly_loop:
662+ paddsw m0, m2, [dstq+strideq*0]
663+ paddsw m1, m2, [dstq+strideq*1]
664+ psubusw m0, m3
665+ psubusw m1, m3
666+ mova [dstq+strideq*0], m0
667+ mova [dstq+strideq*1], m1
668+ lea dstq, [dstq+strideq*2]
669+ dec r3d
670+ jg .dconly_loop
671+ RET
672+.pass1:
673+ vpbroadcastd m20, [o(pd_8192)]
674+ vpbroadcastd m21, [o(pd_11585)]
675+ cmp eobd, 135
676+ jl .pass1_fast
677+ add cq, 64
678+ lea r4, [rsp+64*8]
679+ cmp eobd, 579
680+ jl .pass1_right_fast
681+ mov r6d, 128*28
682+ call .pass1_main
683+ jmp .pass1_right_end
684+.pass1_right_fast: ; bottomright quadrant is zero
685+ mova m0, [cq+128* 1]
686+ mova m1, [cq+128* 3]
687+ mova m2, [cq+128* 5]
688+ mova m3, [cq+128* 7]
689+ mova m4, [cq+128* 9]
690+ mova m5, [cq+128*11]
691+ mova m6, [cq+128*13]
692+ mova m7, [cq+128*15]
693+ call .main_fast
694+ mova m0, [cq+128* 0]
695+ mova m1, [cq+128* 2]
696+ mova m2, [cq+128* 4]
697+ mova m3, [cq+128* 6]
698+ mova m4, [cq+128* 8]
699+ mova m5, [cq+128*10]
700+ mova m6, [cq+128*12]
701+ mova m7, [cq+128*14]
702+ call m(vp9_idct_16x16_internal_10).main_part1_fast
703+ mov r6d, 128*12
704+ call .pass1_main_end
705+.pass1_right_end:
706+ mova [r4+64* 8], m0
707+ mova [r4+64* 9], m1
708+ mova [r4+64*10], m2
709+ mova [r4+64*11], m3
710+ mova [r4+64*12], m4
711+ mova [r4+64*13], m5
712+ mova [r4+64*14], m6
713+ mova [r4+64*15], m7
714+ mova [r4+64*16], m16
715+ mova [r4+64*17], m17
716+ mova [r4+64*18], m18
717+ mova [r4+64*19], m19
718+ mova [r4+64*20], m8
719+ mova [r4+64*21], m9
720+ mova [r4+64*22], m10
721+ mova [r4+64*23], m11
722+ sub cq, 64
723+ sub r4, 64*8
724+ mov r6d, 128*28
725+ call .pass1_main
726+ mova m12, [r4+64*20]
727+ mova m13, [r4+64*21]
728+ mova m14, [r4+64*22]
729+ mova m15, [r4+64*23]
730+ mova [r4+64*20], m8
731+ mova [r4+64*21], m9
732+ mova [r4+64*22], m10
733+ mova [r4+64*23], m11
734+ mova m8, [r4+64*16]
735+ mova m9, [r4+64*17]
736+ mova m10, [r4+64*18]
737+ mova m11, [r4+64*19]
738+ mova [r4+64*16], m16
739+ mova [r4+64*17], m17
740+ mova [r4+64*18], m18
741+ mova [r4+64*19], m19
742+ call .main
743+ mova m0, [r4+64*16]
744+ mova m1, [r4+64*17]
745+ mova m2, [r4+64*18]
746+ mova m3, [r4+64*19]
747+ mova m4, [r4+64*20]
748+ mova m5, [r4+64*21]
749+ mova m6, [r4+64*22]
750+ mova m7, [r4+64*23]
751+ mova m8, [r4+64*24]
752+ mova m9, [r4+64*25]
753+ mova m10, [r4+64*26]
754+ mova m11, [r4+64*27]
755+ mova m12, [r4+64*28]
756+ mova m13, [r4+64*29]
757+ mova m14, [r4+64*30]
758+ mova m15, [r4+64*31]
759+ call m(vp9_idct_16x16_internal_10).main_part1
760+ call .pass2_main_left
761+ mova m8, [r4+64* 8]
762+ mova m9, [r4+64* 9]
763+ mova m10, [r4+64*10]
764+ mova m11, [r4+64*11]
765+ mova m12, [r4+64*12]
766+ mova m13, [r4+64*13]
767+ mova m14, [r4+64*14]
768+ mova m15, [r4+64*15]
769+ TRANSPOSE_4DQ 8, 10, 12, 14, 16
770+ TRANSPOSE_4DQ 9, 11, 13, 15, 16
771+ call .main
772+ call .pass2_main_right
773+ mova m8, [r4+64*24]
774+ mova m9, [r4+64*25]
775+ mova m10, [r4+64*26]
776+ mova m11, [r4+64*27]
777+ mova m12, [r4+64*28]
778+ mova m13, [r4+64*29]
779+ mova m14, [r4+64*30]
780+ mova m15, [r4+64*31]
781+ TRANSPOSE_4DQ 8, 10, 12, 14, 16
782+ TRANSPOSE_4DQ 9, 11, 13, 15, 16
783+ call m(vp9_idct_16x16_internal_10).main_part1
784+ jmp .pass2_end
785+.pass1_fast:
786+ mova m0, [cq+128* 1]
787+ mova m1, [cq+128* 3]
788+ mova m2, [cq+128* 5]
789+ mova m3, [cq+128* 7]
790+ mova m4, [cq+128* 9]
791+ mova m5, [cq+128*11]
792+ mova m6, [cq+128*13]
793+ mova m7, [cq+128*15]
794+ mov r4, rsp
795+ call .main_fast
796+ mova m0, [cq+128* 0]
797+ mova m1, [cq+128* 2]
798+ mova m2, [cq+128* 4]
799+ mova m3, [cq+128* 6]
800+ mova m4, [cq+128* 8]
801+ mova m5, [cq+128*10]
802+ mova m6, [cq+128*12]
803+ mova m7, [cq+128*14]
804+ call m(vp9_idct_16x16_internal_10).main_part1_fast
805+ call m(vp9_idct_16x16_internal_10).main_part2
806+ mov r6d, 128*12
807+ call .pass1_main_end2
808+ mova [r4+64*16], m16
809+ mova [r4+64*17], m17
810+ mova [r4+64*18], m18
811+ mova [r4+64*19], m19
812+ mova [r4+64*20], m8
813+ mova [r4+64*21], m9
814+ mova [r4+64*22], m10
815+ mova [r4+64*23], m11
816+ call .main_fast
817+ mova m0, [r4+64*16]
818+ mova m1, [r4+64*17]
819+ mova m2, [r4+64*18]
820+ mova m3, [r4+64*19]
821+ mova m4, [r4+64*20]
822+ mova m5, [r4+64*21]
823+ mova m6, [r4+64*22]
824+ mova m7, [r4+64*23]
825+ call m(vp9_idct_16x16_internal_10).main_part1_fast
826+ call .pass2_main_left
827+ call .main_fast
828+ call .pass2_main_right
829+ call m(vp9_idct_16x16_internal_10).main_part1_fast
830+.pass2_end:
831+ paddd m0, m22
832+ paddd m18, m22
833+ call m(vp9_idct_16x16_internal_10).main_part2
834+ mova m20, [o(deintq_perm)]
835+ rorx r2, strideq, 59 ; strideq*32
836+ vpbroadcastd m21, [o(pixel_clip)]
837+ add r2, dstq
838+%assign i 0
839+%rep 16
840+ mova m16, [r4+64*(15-i)]
841+ mova m17, [r4+64*(i-16)]
842+ mova m18, [r4-64*(17+i)]
843+ paddd m19, m %+ i, m16
844+ psubd m0, m %+ i, m16
845+ call .write_32x2
846+ %assign i i+1
847+%endrep
848+ RET
849+ALIGN function_align
850+.write_32x2:
851+ paddd m16, m17, m18
852+ psubd m17, m18
853+ REPX {psrad x, 6}, m19, m16, m0, m17
854+ packssdw m16, m19
855+ packssdw m17, m0
856+ sub r2, strideq
857+ vpermq m16, m20, m16
858+ vpermq m17, m20, m17
859+ paddsw m16, [dstq]
860+ paddsw m17, [r2 ]
861+ psubusw m16, m21
862+ psubusw m17, m21
863+ mova [dstq], m16
864+ mova [r2 ], m17
865+ add dstq, strideq
866+ ret
867+ALIGN function_align
868+.pass1_main:
869+ mova m0, [cq+128* 1]
870+ mova m1, [cq+128* 3]
871+ mova m2, [cq+128* 5]
872+ mova m3, [cq+128* 7]
873+ mova m4, [cq+128* 9]
874+ mova m5, [cq+128*11]
875+ mova m6, [cq+128*13]
876+ mova m7, [cq+128*15]
877+ mova m8, [cq+128*17]
878+ mova m9, [cq+128*19]
879+ mova m10, [cq+128*21]
880+ mova m11, [cq+128*23]
881+ mova m12, [cq+128*25]
882+ mova m13, [cq+128*27]
883+ mova m14, [cq+128*29]
884+ mova m15, [cq+128*31]
885+ call .main
886+ mova m0, [cq+128* 0]
887+ mova m1, [cq+128* 2]
888+ mova m2, [cq+128* 4]
889+ mova m3, [cq+128* 6]
890+ mova m4, [cq+128* 8]
891+ mova m5, [cq+128*10]
892+ mova m6, [cq+128*12]
893+ mova m7, [cq+128*14]
894+ mova m8, [cq+128*16]
895+ mova m9, [cq+128*18]
896+ mova m10, [cq+128*20]
897+ mova m11, [cq+128*22]
898+ mova m12, [cq+128*24]
899+ mova m13, [cq+128*26]
900+ mova m14, [cq+128*28]
901+ mova m15, [cq+128*30]
902+ call m(vp9_idct_16x16_internal_10).main_part1
903+.pass1_main_end:
904+ call m(vp9_idct_16x16_internal_10).main_part2
905+.pass1_main_end2:
906+ pxor m16, m16
907+.pass1_zero_loop:
908+ mova [cq+r6+128*0], m16
909+ mova [cq+r6+128*1], m16
910+ mova [cq+r6+128*2], m16
911+ mova [cq+r6+128*3], m16
912+ sub r6d, 128*4
913+ jge .pass1_zero_loop
914+ mova m16, [r4+64*15]
915+ mova m19, [r4+64*14]
916+ mova m22, [r4+64*13]
917+ mova m17, [r4+64*12]
918+ psubd m18, m0, m16
919+ paddd m16, m0
920+ paddd m0, m19, m1
921+ psubd m19, m1, m19
922+ paddd m1, m17, m3
923+ psubd m3, m17
924+ paddd m17, m2, m22
925+ psubd m2, m22
926+ TRANSPOSE_4D 3, 2, 19, 18, 22 ; 28 29 30 31
927+ TRANSPOSE_4D 16, 0, 17, 1, 22 ; 0 1 2 3
928+ mova [r4+64*54], m3
929+ mova [r4+64*55], m19
930+ mova [r4+64*38], m2
931+ mova [r4+64*39], m18
932+ mova m2, [r4+64*11]
933+ mova m19, [r4+64*10]
934+ mova m3, [r4+64* 9]
935+ mova m22, [r4+64* 8]
936+ paddd m18, m4, m2
937+ psubd m4, m2
938+ paddd m2, m5, m19
939+ psubd m5, m19
940+ paddd m19, m6, m3
941+ psubd m6, m3
942+ paddd m3, m7, m22
943+ psubd m7, m22
944+ TRANSPOSE_4D 7, 6, 5, 4, 22 ; 24 25 26 27
945+ TRANSPOSE_4D 18, 2, 19, 3, 22 ; 4 5 6 7
946+ mova [r4+64*52], m7
947+ mova [r4+64*53], m5
948+ mova [r4+64*36], m6
949+ mova [r4+64*37], m4
950+ mova m7, [r4+64* 7]
951+ mova m4, [r4+64* 6]
952+ mova m5, [r4+64* 5]
953+ mova m22, [r4+64* 4]
954+ psubd m6, m8, m7
955+ paddd m8, m7
956+ psubd m7, m9, m4
957+ paddd m4, m9
958+ paddd m9, m10, m5
959+ psubd m10, m5
960+ paddd m5, m11, m22
961+ psubd m11, m22
962+ TRANSPOSE_4D 11, 10, 7, 6, 22 ; 20 21 22 23
963+ TRANSPOSE_4D 8, 4, 9, 5, 22 ; 8 9 10 11
964+ mova [r4+64*50], m11
965+ mova [r4+64*51], m7
966+ mova [r4+64*34], m10
967+ mova [r4+64*35], m6
968+ mova m6, [r4+64* 3]
969+ mova m11, [r4+64* 2]
970+ mova m7, [r4+64* 1]
971+ mova m22, [r4+64* 0]
972+ paddd m10, m12, m6
973+ psubd m12, m6
974+ paddd m6, m13, m11
975+ psubd m13, m11
976+ paddd m11, m14, m7
977+ psubd m14, m7
978+ paddd m7, m15, m22
979+ psubd m15, m22
980+ TRANSPOSE_4D 15, 14, 13, 12, 22 ; 16 17 18 19
981+ TRANSPOSE_4D 10, 6, 11, 7, 22 ; 12 13 14 15
982+ mova [r4+64*48], m15
983+ mova [r4+64*49], m13
984+ mova [r4+64*32], m14
985+ mova [r4+64*33], m12
986+ TRANSPOSE_4DQ 0, 2, 4, 6, 22
987+ TRANSPOSE_4DQ 1, 3, 5, 7, 22
988+ TRANSPOSE_4DQ 16, 18, 8, 10, 22
989+ TRANSPOSE_4DQ 17, 19, 9, 11, 22
990+ ret
991+ALIGN function_align
992+.pass2_main_left:
993+ vpbroadcastd m22, [o(pixel_clip6)]
994+ paddd m0, m22
995+ paddd m18, m22
996+ call m(vp9_idct_16x16_internal_10).main_part2
997+ mova [r4+64*16], m0
998+ mova [r4+64*17], m1
999+ mova [r4+64*18], m2
1000+ mova [r4+64*19], m3
1001+ mova [r4+64*20], m4
1002+ mova [r4+64*21], m5
1003+ mova [r4+64*22], m6
1004+ mova [r4+64*23], m7
1005+ mova [r4+64*24], m8
1006+ mova [r4+64*25], m9
1007+ mova [r4+64*26], m10
1008+ mova [r4+64*27], m11
1009+ mova [r4+64*28], m12
1010+ mova [r4+64*29], m13
1011+ mova [r4+64*30], m14
1012+ mova [r4+64*31], m15
1013+ add r4, 64*32
1014+ mova m0, [r4+64* 0]
1015+ mova m1, [r4+64* 1]
1016+ mova m2, [r4+64* 2]
1017+ mova m3, [r4+64* 3]
1018+ mova m4, [r4+64* 4]
1019+ mova m5, [r4+64* 5]
1020+ mova m6, [r4+64* 6]
1021+ mova m7, [r4+64* 7]
1022+ jmp .pass2_main_transpose
1023+ALIGN function_align
1024+.pass2_main_right:
1025+ mova m0, [r4+64*16]
1026+ mova m1, [r4+64*17]
1027+ mova m2, [r4+64*18]
1028+ mova m3, [r4+64*19]
1029+ mova m4, [r4+64*20]
1030+ mova m5, [r4+64*21]
1031+ mova m6, [r4+64*22]
1032+ mova m7, [r4+64*23]
1033+.pass2_main_transpose:
1034+ TRANSPOSE_4DQ 0, 2, 4, 6, 8
1035+ TRANSPOSE_4DQ 1, 3, 5, 7, 8
1036+ ret
1037+ALIGN function_align
1038+.main_fast:
1039+ pmulld m15, m0, [o(pd_16364)] {1to16} ; t31a
1040+ pmulld m0, [o(pd_804)] {1to16} ; t16a
1041+ pmulld m8, m7, [o(pd_11003)] {1to16} ; t17a
1042+ pmulld m7, [o(pd_12140)] {1to16} ; t30a
1043+ pmulld m11, m4, [o(pd_14811)] {1to16} ; t29a
1044+ pmulld m4, [o(pd_7005)] {1to16} ; t18a
1045+ pmulld m12, m3, [o(pd_5520)] {1to16} ; t19a
1046+ pmulld m3, [o(pd_15426)] {1to16} ; t28a
1047+ pmulld m13, m2, [o(pd_15893)] {1to16} ; t27a
1048+ pmulld m2, [o(pd_3981)] {1to16} ; t20a
1049+ pmulld m10, m5, [o(pd_8423)] {1to16} ; t21a
1050+ pmulld m5, [o(pd_14053)] {1to16} ; t26a
1051+ pmulld m9, m6, [o(pd_13160)] {1to16} ; t25a
1052+ pmulld m6, [o(pd_9760)] {1to16} ; t22a
1053+ pmulld m14, m1, [o(pd_2404)] {1to16} ; t23a
1054+ pmulld m1, [o(pd_16207)] {1to16} ; t24a
1055+ REPX {psubd x, m20, x}, m8, m12, m10, m14
1056+ jmp .main2
1057+ALIGN function_align
1058+.main:
1059+ ITX_MULSUB_2D 0, 15, 16, 17, 18, _, 804, 16364 ; t16a, t31a
1060+ ITX_MULSUB_2D 8, 7, 16, 17, 18, _, 12140, 11003 ; t17a, t30a
1061+ ITX_MULSUB_2D 4, 11, 16, 17, 18, _, 7005, 14811 ; t18a, t29a
1062+ ITX_MULSUB_2D 12, 3, 16, 17, 18, _, 15426, 5520 ; t19a, t28a
1063+ ITX_MULSUB_2D 2, 13, 16, 17, 18, _, 3981, 15893 ; t20a, t27a
1064+ ITX_MULSUB_2D 10, 5, 16, 17, 18, _, 14053, 8423 ; t21a, t26a
1065+ ITX_MULSUB_2D 6, 9, 16, 17, 18, _, 9760, 13160 ; t22a, t25a
1066+ ITX_MULSUB_2D 14, 1, 16, 17, 18, _, 16207, 2404 ; t23a, t24a
1067+ REPX {paddd x, m20}, m8, m12, m10, m14
1068+.main2:
1069+ REPX {paddd x, m20}, m0, m15, m7, m4, m3, m11
1070+ REPX {psrad x, 14 }, m8, m0, m15, m7, m12, m4, m3, m11
1071+ psubd m16, m0, m8 ; t17
1072+ paddd m0, m8 ; t16
1073+ psubd m8, m15, m7 ; t30
1074+ paddd m15, m7 ; t31
1075+ paddd m7, m12, m4 ; t19
1076+ psubd m12, m4 ; t18
1077+ paddd m4, m3, m11 ; t28
1078+ psubd m3, m11 ; t29
1079+ REPX {paddd x, m20}, m2, m13, m5, m6, m1, m9
1080+ REPX {psrad x, 14 }, m10, m2, m13, m5, m14, m6, m1, m9
1081+ psubd m11, m2, m10 ; t21
1082+ paddd m2, m10 ; t20
1083+ psubd m10, m13, m5 ; t26
1084+ paddd m13, m5 ; t27
1085+ psubd m5, m14, m6 ; t22
1086+ paddd m6, m14 ; t23
1087+ psubd m14, m1, m9 ; t25
1088+ paddd m9, m1 ; t24
1089+ vpbroadcastd m19, [o(pd_16069)]
1090+ vpbroadcastd m18, [o(pd_3196)]
1091+ ITX_MULSUB_2D 8, 16, 1, 17, _, 20, 18, 19 ; t17a, t30a
1092+ ITX_MULSUB_2D 3, 12, 1, 17, _, 20, 18, 19, 1 ; t29a, t18a
1093+ vpbroadcastd m19, [o(pd_9102)]
1094+ vpbroadcastd m18, [o(pd_13623)]
1095+ ITX_MULSUB_2D 10, 11, 1, 17, _, 20, 18, 19 ; t21a, t26a
1096+ ITX_MULSUB_2D 14, 5, 1, 17, _, 20, 18, 19, 1 ; t25a, t22a
1097+ paddd m1, m6, m2 ; t23a
1098+ psubd m6, m2 ; t20a
1099+ psubd m2, m9, m13 ; t27a
1100+ paddd m9, m13 ; t24a
1101+ psubd m13, m15, m4 ; t28a
1102+ paddd m15, m4 ; t31a
1103+ psubd m4, m8, m12 ; t18
1104+ paddd m8, m12 ; t17
1105+ psubd m12, m0, m7 ; t19a
1106+ paddd m0, m7 ; t16a
1107+ psubd m7, m16, m3 ; t29
1108+ paddd m3, m16 ; t30
1109+ paddd m16, m5, m10 ; t22
1110+ psubd m5, m10 ; t21
1111+ psubd m10, m14, m11 ; t26
1112+ paddd m14, m11 ; t25
1113+ vpbroadcastd m19, [o(pd_15137)]
1114+ vpbroadcastd m18, [o(pd_6270)]
1115+ ITX_MULSUB_2D 13, 12, 11, 17, _, 20, 18, 19 ; t19, t28
1116+ ITX_MULSUB_2D 2, 6, 11, 17, _, 20, 18, 19, 1 ; t27, t20
1117+ ITX_MULSUB_2D 7, 4, 11, 17, _, 20, 18, 19 ; t18a, t29a
1118+ ITX_MULSUB_2D 10, 5, 11, 17, _, 20, 18, 19, 1 ; t26a, t21a
1119+ psubd m11, m0, m1 ; t23
1120+ paddd m0, m1 ; t16
1121+ paddd m1, m16, m8 ; t17a
1122+ psubd m16, m8, m16 ; t22a
1123+ psubd m8, m15, m9 ; t24
1124+ paddd m15, m9 ; t31
1125+ psubd m9, m3, m14 ; t25a
1126+ paddd m14, m3 ; t30a
1127+ paddd m3, m6, m13 ; t19a
1128+ psubd m6, m13, m6 ; t20a
1129+ paddd m13, m10, m4 ; t29
1130+ psubd m10, m4, m10 ; t26
1131+ psubd m4, m12, m2 ; t27a
1132+ paddd m12, m2 ; t28a
1133+ paddd m2, m7, m5 ; t18
1134+ psubd m7, m5 ; t21
1135+ REPX {pmulld x, m21}, m10, m8, m4, m9, m7, m11, m6, m16
1136+ mova [r4+64* 0], m0
1137+ mova [r4+64* 1], m1
1138+ mova [r4+64* 2], m2
1139+ mova [r4+64* 3], m3
1140+ mova [r4+64*12], m12
1141+ mova [r4+64*13], m13
1142+ mova [r4+64*14], m14
1143+ mova [r4+64*15], m15
1144+ REPX {paddd x, m20}, m10, m8, m4, m9
1145+ psubd m5, m10, m7 ; t21a
1146+ paddd m10, m7 ; t26a
1147+ psubd m7, m8, m11 ; t23a
1148+ paddd m8, m11 ; t24a
1149+ REPX {psrad x, 14 }, m5, m10, m7, m8
1150+ paddd m11, m4, m6 ; t27
1151+ psubd m4, m6 ; t20
1152+ psubd m6, m9, m16 ; t22
1153+ paddd m9, m16 ; t25
1154+ REPX {psrad x, 14 }, m11, m4, m6, m9
1155+ mova [r4+64* 4], m4
1156+ mova [r4+64* 5], m5
1157+ mova [r4+64* 6], m6
1158+ mova [r4+64* 7], m7
1159+ mova [r4+64* 8], m8
1160+ mova [r4+64* 9], m9
1161+ mova [r4+64*10], m10
1162+ mova [r4+64*11], m11
1163+ ret
1164+
1165+%endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t