History

avcodec/x86/vp9: Add AVX-512ICL for 16x16 and 32x32 10bpc inverse transforms

Henrik Gramner committed 1 year ago

Commit eda0ac71 parent df967d04 files · +1176 −2Browse files

Showing the changes to libavcodec (4 of 4 files). Show all

Unified · Split

4 files
  1. libavcodec/vp9dec.h +2 −2
  2. libavcodec/x86/Makefile +1 −0
  3. libavcodec/x86/vp9dsp_init_16bpp_template.c +8 −0
  4. libavcodec/x86/vp9itxfm_16bpp_avx512.asm +1165 −0
libavcodec/vp9dec.h+2−2
@@ -220,8 +220,8 @@ struct VP9TileData { · 219 unmodified lines
220 DECLARE_ALIGNED(8, uint8_t, left_ref_ctx)[8];220 DECLARE_ALIGNED(8, uint8_t, left_ref_ctx)[8];
221 DECLARE_ALIGNED(8, uint8_t, left_filter_ctx)[8];221 DECLARE_ALIGNED(8, uint8_t, left_filter_ctx)[8];
222 // block reconstruction intermediates222 // block reconstruction intermediates
223 DECLARE_ALIGNED(32, uint8_t, tmp_y)[64 * 64 * 2];223 DECLARE_ALIGNED(64, uint8_t, tmp_y)[64 * 64 * 2];
224 DECLARE_ALIGNED(32, uint8_t, tmp_uv)[2][64 * 64 * 2];224 DECLARE_ALIGNED(64, uint8_t, tmp_uv)[2][64 * 64 * 2];
225 struct { int x, y; } min_mv, max_mv;225 struct { int x, y; } min_mv, max_mv;
226 int16_t *block_base, *block, *uvblock_base[2], *uvblock[2];226 int16_t *block_base, *block, *uvblock_base[2], *uvblock[2];
227 uint8_t *eob_base, *uveob_base[2], *eob, *uveob[2];227 uint8_t *eob_base, *uveob_base[2], *eob, *uveob[2];
libavcodec/x86/Makefile+1−0
@@ -186,6 +186,7 @@ X86ASM-OBJS-$(CONFIG_VP9_DECODER) += x86/vp9intrapred.o \ · 185 unmodified lines
186 x86/vp9itxfm.o \186 x86/vp9itxfm.o \
187 x86/vp9itxfm_avx512.o \187 x86/vp9itxfm_avx512.o \
188 x86/vp9itxfm_16bpp.o \188 x86/vp9itxfm_16bpp.o \
189 x86/vp9itxfm_16bpp_avx512.o \
189 x86/vp9lpf.o \190 x86/vp9lpf.o \
190 x86/vp9lpf_16bpp.o \191 x86/vp9lpf_16bpp.o \
191 x86/vp9mc.o \192 x86/vp9mc.o \
libavcodec/x86/vp9dsp_init_16bpp_template.c+8−0
@@ -127,6 +127,8 @@ decl_itxfm_func(iwht, iwht, 4, BPC, mmxext); · 126 unmodified lines
127#if BPC == 10127#if BPC == 10
128decl_itxfm_func(idct, idct, 4, BPC, mmxext);128decl_itxfm_func(idct, idct, 4, BPC, mmxext);
129decl_itxfm_funcs(4, BPC, ssse3);129decl_itxfm_funcs(4, BPC, ssse3);
130decl_itxfm_funcs(16, BPC, avx512icl);
131decl_itxfm_func(idct, idct, 32, BPC, avx512icl);
130#else132#else
131decl_itxfm_func(idct, idct, 4, BPC, sse2);133decl_itxfm_func(idct, idct, 4, BPC, sse2);
132#endif134#endif
@@ -233,6 +235,12 @@ av_cold void INIT_FUNC(VP9DSPContext *dsp, int bitexact) · 100 unmodified lines
233#endif235#endif
234 }236 }
235 237
238#if ARCH_X86_64 && BPC == 10
239 if (EXTERNAL_AVX512ICL(cpu_flags)) {
240 init_itx_funcs(TX_16X16, 16, BPC, avx512icl);
241 init_itx_func_one(TX_32X32, idct, idct, 32, BPC, avx512icl);
242 }
243#endif
236#endif /* HAVE_X86ASM */244#endif /* HAVE_X86ASM */
237 245
238 ff_vp9dsp_init_16bpp_x86(dsp);246 ff_vp9dsp_init_16bpp_x86(dsp);
libavcodec/x86/vp9itxfm_16bpp_avx512.asm+1165−0
@@ -0,0 +1,1165 @@
1;******************************************************************************
2;* VP9 IDCT SIMD optimizations
3;*
4;* Copyright (C) 2025 Two Orioles, LLC
5;*
6;* This file is part of FFmpeg.
7;*
8;* FFmpeg is free software; you can redistribute it and/or
9;* modify it under the terms of the GNU Lesser General Public
10;* License as published by the Free Software Foundation; either
11;* version 2.1 of the License, or (at your option) any later version.
12;*
13;* FFmpeg is distributed in the hope that it will be useful,
14;* but WITHOUT ANY WARRANTY; without even the implied warranty of
15;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16;* Lesser General Public License for more details.
17;*
18;* You should have received a copy of the GNU Lesser General Public
19;* License along with FFmpeg; if not, write to the Free Software
20;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21;******************************************************************************
22
23%include "libavutil/x86/x86util.asm"
24
25%if ARCH_X86_64 && HAVE_AVX512ICL_EXTERNAL
26
27SECTION_RODATA 64
28
29; Thw following set of constants are ordered to form the
30; qword shuffle mask { 0, 2, 4, 6, 1, 3, 5, 7 }
31%define deintq_perm pd_5520
32pd_5520: dd 5520
33pd_9760: dd 9760
34pd_10394: dd 10394
35pd_15426: dd 15426
36pd_804: dd 804
37pd_2404: dd 2404
38pd_6270: dd 6270
39pd_9102: dd 9102
40pd_11585: dd 11585
41pd_12665: dd 12665
42pd_7723: dd 7723
43pd_14811: dd 14811
44pd_7005: dd 7005
45pd_14053: dd 14053
46pd_8423: dd 8423
47pd_13623: dd 13623
48
49pixel_clip: times 2 dw 0x7c00
50pixel_clip6: dd 2031648 ; 32 + (pixel_clip << 6)
51pd_532480: dd 532480 ; 8192 + (32 << 14)
52pd_8192: dd 8192
53
54pd_1606: dd 1606
55pd_3196: dd 3196
56pd_3981: dd 3981
57pd_4756: dd 4756
58pd_11003: dd 11003
59pd_12140: dd 12140
60pd_13160: dd 13160
61pd_14449: dd 14449
62pd_15137: dd 15137
63pd_15679: dd 15679
64pd_15893: dd 15893
65pd_16069: dd 16069
66pd_16207: dd 16207
67pd_16305: dd 16305
68pd_16364: dd 16364
69
70SECTION .text
71
72%define o_base (deintq_perm+128)
73%define o(x) (r5 - o_base + (x))
74%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)
75
76; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12
77; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12
78; skip round/shift if rnd is not a number
79%macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], inv_dst2
80%if %8 < 32
81 pmulld m%4, m%1, m%8
82 pmulld m%3, m%2, m%8
83%else
84 vpbroadcastd m%3, [o(pd_%8)]
85 pmulld m%4, m%1, m%3
86 pmulld m%3, m%2
87%endif
88%if %7 < 32
89 pmulld m%1, m%7
90 pmulld m%2, m%7
91%else
92 vpbroadcastd m%5, [o(pd_%7)]
93 pmulld m%1, m%5
94 pmulld m%2, m%5
95%endif
96%if %9
97 psubd m%4, m%6, m%4
98 psubd m%2, m%4, m%2
99%else
100%ifnum %6
101 paddd m%4, m%6
102%endif
103 paddd m%2, m%4
104%endif
105%ifnum %6
106 paddd m%1, m%6
107%endif
108 psubd m%1, m%3
109%ifnum %6
110 psrad m%2, 14
111 psrad m%1, 14
112%endif
113%endmacro
114
115%macro WRAP_YMM 1+
116 INIT_YMM cpuname
117 %1
118 INIT_ZMM cpuname
119%endmacro
120
121%macro TRANSPOSE_4D 5 ; in[1-4], tmp
122 punpckhdq m%5, m%3, m%4 ; c2 d2 c3 d3
123 punpckldq m%3, m%4 ; c0 d0 c1 d1
124 punpckhdq m%4, m%1, m%2 ; a2 b2 a3 b3
125 punpckldq m%1, m%2 ; a0 b0 a1 b1
126 punpckhqdq m%2, m%1, m%3 ; a1 b1 c1 d1
127 punpcklqdq m%1, m%3 ; a0 b0 c0 d0
128 punpcklqdq m%3, m%4, m%5 ; a2 b2 c2 d2
129 punpckhqdq m%4, m%5 ; a3 b3 c3 d3
130%endmacro
131
132%macro TRANSPOSE_4DQ 5 ; in[1-4], tmp
133 vshufi32x4 m%5, m%3, m%4, q3232 ; c2 c3 d2 d3
134 vinserti32x8 m%3, ym%4, 1 ; c0 c1 d0 d1
135 vshufi32x4 m%4, m%1, m%2, q3232 ; a2 a3 b2 b3
136 vinserti32x8 m%1, ym%2, 1 ; a0 a1 b0 b1
137 vshufi32x4 m%2, m%1, m%3, q3131 ; a1 b1 c1 d1
138 vshufi32x4 m%1, m%3, q2020 ; a0 b0 c0 d0
139 vshufi32x4 m%3, m%4, m%5, q2020 ; a2 b2 c2 d2
140 vshufi32x4 m%4, m%5, q3131 ; a3 b3 c3 d3
141%endmacro
142
143%macro INV_TXFM_FN 3-4 0 ; type1, type2, size, eob_offset
144cglobal vp9_i%1_i%2_%3_add_10, 4, 5, 0, dst, stride, c, eob, tx2
145 %define %%p1 m(vp9_i%1_%3_internal_10)
146 lea r5, [o_base]
147 ; Jump to the 1st txfm function if we're not taking the fast path, which
148 ; in turn performs an indirect jump to the 2nd txfm function.
149 lea tx2q, [m(vp9_i%2_%3_internal_10).pass2]
150%ifidn %1_%2, dct_dct
151 dec eobd
152 jnz %%p1
153%else
154%if %4
155 add eobd, %4
156%endif
157 ; jump to the 1st txfm function unless it's located directly after this
158 times ((%%end - %%p1) >> 31) & 1 jmp %%p1
159ALIGN function_align
160%%end:
161%endif
162%endmacro
163
164%macro INV_TXFM_16X16_FN 2-3 0 ; type1, type2, eob_offset
165 INV_TXFM_FN %1, %2, 16x16, %3
166%ifidn %1_%2, dct_dct
167 imul r6d, [cq], 11585
168 vpbroadcastd ym3, [o(pixel_clip)]
169 mov [cq], r3d
170 add r6d, 8192
171 sar r6d, 14
172 imul r6d, 11585
173 or r3d, 8
174 add r6d, 532480
175 sar r6d, 20
176 vpbroadcastw ym2, r6d
177 paddsw ym2, ym3
178.dconly_loop:
179 paddsw ym0, ym2, [dstq+strideq*0]
180 paddsw ym1, ym2, [dstq+strideq*1]
181 psubusw ym0, ym3
182 psubusw ym1, ym3
183 mova [dstq+strideq*0], ym0
184 mova [dstq+strideq*1], ym1
185 lea dstq, [dstq+strideq*2]
186 dec r3d
187 jg .dconly_loop
188 RET
189%endif
190%endmacro
191
192%macro IDCT16_PART1 0
193%if mmsize == 64
194.main_part1_fast:
195%endif
196 pmulld m15, m1, [o(pd_16305)] {bcstd} ; t15a
197 pmulld m1, [o(pd_1606)] {bcstd} ; t8a
198 pmulld m9, m7, [o(pd_10394)] {bcstd} ; t9a
199 pmulld m7, [o(pd_12665)] {bcstd} ; t14a
200 pmulld m11, m5, [o(pd_14449)] {bcstd} ; t13a
201 pmulld m5, [o(pd_7723)] {bcstd} ; t10a
202 pmulld m13, m3, [o(pd_4756)] {bcstd} ; t11a
203 pmulld m3, [o(pd_15679)] {bcstd} ; t12a
204 pmulld m10, m6, [o(pd_9102)] {bcstd} ; t5a
205 pmulld m6, [o(pd_13623)] {bcstd} ; t6a
206 pmulld m14, m2, [o(pd_16069)] {bcstd} ; t7a
207 pmulld m2, [o(pd_3196)] {bcstd} ; t4a
208 pmulld m12, m4, [o(pd_15137)] {bcstd} ; t3
209 pmulld m4, [o(pd_6270)] {bcstd} ; t2
210 pmulld m0, m21
211 REPX {psubd x, m20, x}, m9, m13, m10
212 paddd m0, m20
213 mova m18, m0
214%if mmsize == 64 ; for the ymm variant we only ever use the fast path
215 jmp %%main_part1b
216.main_part1:
217 ITX_MULSUB_2D 1, 15, 16, 17, 18, _, 1606, 16305 ; t8a, t15a
218 ITX_MULSUB_2D 9, 7, 16, 17, 18, _, 12665, 10394 ; t9a, t14a
219 ITX_MULSUB_2D 5, 11, 16, 17, 18, _, 7723, 14449 ; t10a, t13a
220 ITX_MULSUB_2D 13, 3, 16, 17, 18, _, 15679, 4756 ; t11a, t12a
221 ITX_MULSUB_2D 10, 6, 16, 17, 18, _, 13623, 9102 ; t5a, t6a
222 ITX_MULSUB_2D 2, 14, 16, 17, 18, _, 3196, 16069 ; t4a, t7a
223 ITX_MULSUB_2D 4, 12, 16, 17, 18, _, 6270, 15137 ; t2, t3
224 pmulld m0, m21
225 pmulld m8, m21
226 REPX {paddd x, m20}, m0, m9, m13, m10
227 psubd m18, m0, m8 ; t1
228 paddd m0, m8 ; t0
229%%main_part1b:
230%endif
231 vpbroadcastd m19, [o(pd_15137)]
232 vpbroadcastd m16, [o(pd_6270)]
233 REPX {paddd x, m20}, m15, m7, m1, m11, m3, m5
234 REPX {psrad x, 14 }, m15, m7, m1, m9, m11, m3, m5, m13
235 paddd m17, m15, m7 ; t15
236 psubd m15, m7 ; t14
237 psubd m7, m3, m11 ; t13
238 paddd m3, m11 ; t12
239 psubd m11, m13, m5 ; t10
240 paddd m5, m13 ; t11
241 psubd m13, m1, m9 ; t9
242 paddd m1, m9 ; t8
243 ITX_MULSUB_2D 15, 13, 8, 9, _, 20, 16, 19 ; t9a, t14a
244 ITX_MULSUB_2D 7, 11, 8, 9, _, 20, 16, 19, 2 ; t13a, t10a
245 paddd m16, m1, m5 ; t8a
246 psubd m1, m5 ; t11a
247 paddd m8, m15, m11 ; t9
248 psubd m15, m11 ; t10
249 psubd m11, m17, m3 ; t12a
250 paddd m17, m3 ; t15a
251 psubd m9, m13, m7 ; t13
252 paddd m13, m7 ; t14
253 REPX {pmulld x, m21}, m11, m9, m1, m15
254 REPX {paddd x, m20}, m2, m6, m14
255 REPX {psrad x, 14 }, m10, m2, m6, m14
256 psubd m3, m2, m10 ; t5a
257 paddd m10, m2 ; t4
258 paddd m11, m20
259 psubd m5, m11, m1 ; t11
260 paddd m11, m1 ; t12
261 psubd m1, m14, m6 ; t6a
262 paddd m14, m6 ; t7
263 pmulld m1, m21
264 pmulld m3, m21
265 paddd m4, m20
266 paddd m12, m20
267 REPX {psrad x, 14 }, m4, m12, m0, m18
268 paddd m9, m20
269 paddd m2, m9, m15 ; t13a
270 psubd m9, m15 ; t10a
271 paddd m1, m20
272 psubd m6, m1, m3 ; t5
273 paddd m1, m3 ; t6
274 REPX {psrad x, 14}, m6, m1, m11, m5, m2, m9
275%endmacro
276
277%macro IDCT16_PART2 0
278 psubd m3, m0, m12 ; t3
279 paddd m0, m12 ; t0
280 psubd m12, m18, m4 ; t2
281 paddd m18, m4 ; t1
282 psubd m4, m3, m10 ; t4
283 paddd m3, m10 ; t3
284 psubd m10, m12, m6 ; t5
285 paddd m12, m6 ; t2
286 psubd m6, m18, m1 ; t6
287 paddd m1, m18 ; t1
288 psubd m7, m0, m14 ; t7
289 paddd m0, m14 ; t0
290 psubd m15, m0, m17 ; out15
291 paddd m0, m17 ; out0
292 psubd m14, m1, m13 ; out14
293 paddd m1, m13 ; out1
294 psubd m13, m12, m2 ; out13
295 paddd m2, m12 ; out2
296 psubd m12, m3, m11 ; out12
297 paddd m3, m11 ; out3
298 psubd m11, m4, m5 ; out11
299 paddd m4, m5 ; out4
300 paddd m5, m10, m9 ; out5
301 psubd m10, m9 ; out10
302 psubd m9, m6, m8 ; out9
303 paddd m6, m8 ; out6
304 psubd m8, m7, m16 ; out8
305 paddd m7, m16 ; out7
306%endmacro
307
308INIT_ZMM avx512icl
309INV_TXFM_16X16_FN dct, dct
310INV_TXFM_16X16_FN dct, adst, 39-23-1
311
312cglobal vp9_idct_16x16_internal_10, 0, 7, 22, dst, stride, c, eob, tx2
313 mova m0, [cq+64* 0]
314 mova m1, [cq+64* 1]
315 mova m2, [cq+64* 2]
316 mova m3, [cq+64* 3]
317 mova m4, [cq+64* 4]
318 mova m5, [cq+64* 5]
319 mova m6, [cq+64* 6]
320 mova m7, [cq+64* 7]
321 vpbroadcastd m20, [o(pd_8192)]
322 vpbroadcastd m21, [o(pd_11585)]
323 sub eobd, 38
324 jl .pass1_fast
325 mova m8, [cq+64* 8]
326 mova m9, [cq+64* 9]
327 mova m10, [cq+64*10]
328 mova m11, [cq+64*11]
329 mova m12, [cq+64*12]
330 mova m13, [cq+64*13]
331 mova m14, [cq+64*14]
332 mova m15, [cq+64*15]
333 call .main_part1
334 call .main_part2
335.pass1_end:
336 TRANSPOSE_4DQ 0, 4, 8, 12, 16
337 TRANSPOSE_4DQ 1, 5, 9, 13, 16
338 TRANSPOSE_4DQ 2, 6, 10, 14, 16
339 TRANSPOSE_4DQ 3, 7, 11, 15, 16
340 TRANSPOSE_4D 8, 9, 10, 11, 16
341 TRANSPOSE_4D 12, 13, 14, 15, 16
342 mov r6d, 64*12
343 jmp .pass1_transpose_end
344.pass1_fast:
345 WRAP_YMM IDCT16_PART1
346 WRAP_YMM IDCT16_PART2
347.pass1_fast_end:
348 vinserti32x8 m0, ym4, 1
349 vinserti32x8 m8, ym12, 1
350 vinserti32x8 m1, ym5, 1
351 vinserti32x8 m9, ym13, 1
352 vinserti32x8 m2, ym6, 1
353 vinserti32x8 m10, ym14, 1
354 vinserti32x8 m3, ym7, 1
355 vinserti32x8 m11, ym15, 1
356 vshufi32x4 m4, m0, m8, q3131
357 vshufi32x4 m0, m8, q2020
358 vshufi32x4 m5, m1, m9, q3131
359 vshufi32x4 m1, m9, q2020
360 vshufi32x4 m6, m2, m10, q3131
361 vshufi32x4 m2, m10, q2020
362 vshufi32x4 m7, m3, m11, q3131
363 vshufi32x4 m3, m11, q2020
364 mov r6d, 64*4
365.pass1_transpose_end:
366 pxor m16, m16
367.zero_loop:
368 mova [cq+r6+64*0], m16
369 mova [cq+r6+64*1], m16
370 mova [cq+r6+64*2], m16
371 mova [cq+r6+64*3], m16
372 sub r6d, 64*4
373 jge .zero_loop
374 TRANSPOSE_4D 0, 1, 2, 3, 16
375 TRANSPOSE_4D 4, 5, 6, 7, 16
376 jmp tx2q
377.pass2:
378 test eobd, eobd
379 jl .pass2_fast
380 call .main_part1
381 jmp .pass2_end
382.pass2_fast:
383 call .main_part1_fast
384.pass2_end:
385 vpbroadcastd m3, [o(pixel_clip6)]
386 paddd m0, m3
387 paddd m18, m3
388 call .main_part2
389 REPX {psrad x, 6}, m0, m1, m2, m3
390 packssdw m0, m1
391 lea r6, [strideq*3]
392 packssdw m1, m2, m3
393 mova m2, [o(deintq_perm)]
394 vpbroadcastd m3, [o(pixel_clip)]
395 REPX {psrad x, 6}, m4, m5, m6, m7
396 call .write_16x4
397 packssdw m0, m4, m5
398 packssdw m1, m6, m7
399 REPX {psrad x, 6}, m8, m9, m10, m11
400 call .write_16x4
401 packssdw m0, m8, m9
402 packssdw m1, m10, m11
403.pass2_end2:
404 REPX {psrad x, 6}, m12, m13, m14, m15
405 call .write_16x4
406 packssdw m0, m12, m13
407 packssdw m1, m14, m15
408 call .write_16x4
409 RET
410ALIGN function_align
411.write_16x4:
412 mova ym16, [dstq+strideq*0]
413 vinserti32x8 m16, [dstq+strideq*1], 1
414 mova ym17, [dstq+strideq*2]
415 vinserti32x8 m17, [dstq+r6 ], 1
416 vpermq m0, m2, m0
417 vpermq m1, m2, m1
418 paddsw m16, m0
419 paddsw m17, m1
420 psubusw m16, m3
421 psubusw m17, m3
422 mova [dstq+strideq*0], ym16
423 vextracti32x8 [dstq+strideq*1], m16, 1
424 mova [dstq+strideq*2], ym17
425 vextracti32x8 [dstq+r6 ], m17, 1
426 lea dstq, [dstq+strideq*4]
427 ret
428ALIGN function_align
429 IDCT16_PART1
430 ret
431ALIGN function_align
432.main_part2:
433 IDCT16_PART2
434 ret
435
436%macro IADST16_PART1 0
437%if mmsize == 64
438.main_part1_fast:
439%endif
440 pmulld m15, m0, [o(pd_16364)] {bcstd} ; t1
441 pmulld m0, [o(pd_804)] {bcstd} ; t0
442 pmulld m13, m2, [o(pd_15893)] {bcstd} ; t3
443 pmulld m2, [o(pd_3981)] {bcstd} ; t2
444 pmulld m11, m4, [o(pd_14811)] {bcstd} ; t5
445 pmulld m4, [o(pd_7005)] {bcstd} ; t4
446 pmulld m9, m6, [o(pd_13160)] {bcstd} ; t7
447 pmulld m6, [o(pd_9760)] {bcstd} ; t6
448 pmulld m8, m7, [o(pd_11003)] {bcstd} ; t8
449 pmulld m7, [o(pd_12140)] {bcstd} ; t9
450 pmulld m10, m5, [o(pd_8423)] {bcstd} ; t10
451 pmulld m5, [o(pd_14053)] {bcstd} ; t11
452 pmulld m12, m3, [o(pd_5520)] {bcstd} ; t12
453 pmulld m3, [o(pd_15426)] {bcstd} ; t13
454 pmulld m14, m1, [o(pd_2404)] {bcstd} ; t14
455 pmulld m1, [o(pd_16207)] {bcstd} ; t15
456 REPX {psubd x, m20, x}, m15, m13, m11, m9
457%if mmsize == 64 ; for the ymm variant we only ever use the fast path
458 jmp %%main_part1b
459ALIGN function_align
460.main_part1:
461 ITX_MULSUB_2D 15, 0, 16, 17, 18, _, 804, 16364 ; t1, t0
462 ITX_MULSUB_2D 13, 2, 16, 17, 18, _, 3981, 15893 ; t3, t2
463 ITX_MULSUB_2D 11, 4, 16, 17, 18, _, 7005, 14811 ; t5, t4
464 ITX_MULSUB_2D 9, 6, 16, 17, 18, _, 9760, 13160 ; t7, t6
465 ITX_MULSUB_2D 7, 8, 16, 17, 18, _, 12140, 11003 ; t9, t8
466 ITX_MULSUB_2D 5, 10, 16, 17, 18, _, 14053, 8423 ; t11, t10
467 ITX_MULSUB_2D 3, 12, 16, 17, 18, _, 15426, 5520 ; t13, t12
468 ITX_MULSUB_2D 1, 14, 16, 17, 18, _, 16207, 2404 ; t15, t14
469 REPX {paddd x, m20}, m15, m13, m11, m9
470%%main_part1b:
471%endif
472 REPX {paddd x, m20}, m0, m2, m4, m6
473 psubd m16, m2, m10 ; t10a
474 paddd m2, m10 ; t2a
475 psubd m10, m9, m1 ; t15a
476 paddd m9, m1 ; t7a
477 psubd m1, m13, m5 ; t11a
478 paddd m13, m5 ; t3a
479 psubd m5, m6, m14 ; t14a
480 paddd m6, m14 ; t6a
481 REPX {psrad x, 14}, m16, m10, m1, m5
482 psubd m14, m0, m8 ; t8a
483 paddd m0, m8 ; t0a
484 psubd m8, m15, m7 ; t9a
485 paddd m15, m7 ; t1a
486 psubd m7, m4, m12 ; t12a
487 paddd m4, m12 ; t4a
488 paddd m12, m11, m3 ; t5a
489 psubd m11, m3 ; t13a
490 REPX {psrad x, 14}, m14, m8, m7, m11
491 vpbroadcastd m19, [o(pd_9102)]
492 vpbroadcastd m18, [o(pd_13623)]
493 ITX_MULSUB_2D 16, 1, 3, 17, _, _, 18, 19 ; t11, t10
494 ITX_MULSUB_2D 10, 5, 3, 17, _, _, 19, 18 ; t14, t15
495 vpbroadcastd m19, [o(pd_16069)]
496 vpbroadcastd m18, [o(pd_3196)]
497 ITX_MULSUB_2D 14, 8, 3, 17, _, _, 18, 19 ; t9, t8
498 ITX_MULSUB_2D 11, 7, 3, 17, _, _, 19, 18 ; t12, t13
499 vpbroadcastd m19, [o(pd_6270)]
500 vpbroadcastd m18, [o(pd_15137)]
501 REPX {psrad x, 14}, m15, m12, m0, m4
502 psubd m3, m15, m12 ; t5
503 paddd m15, m12 ; t1
504 psubd m12, m0, m4 ; t4
505 paddd m0, m4 ; t0
506 REPX {psrad x, 14}, m2, m6, m13, m9
507 psubd m4, m2, m6 ; t6
508 paddd m2, m6 ; t2
509 psubd m6, m13, m9 ; t7
510 paddd m9, m13 ; t3
511 REPX {paddd x, m20}, m8, m14, m1, m16
512 psubd m13, m8, m11 ; t12a
513 paddd m8, m11 ; t8a
514 psubd m11, m14, m7 ; t13a
515 paddd m14, m7 ; t9a
516 psubd m7, m1, m10 ; t14a
517 paddd m1, m10 ; t10a
518 psubd m10, m16, m5 ; t15a
519 paddd m16, m5 ; t11a
520 REPX {psrad x, 14}, m13, m11, m7, m10
521 ITX_MULSUB_2D 12, 3, 5, 17, _, _, 19, 18 ; t5a, t4a
522 ITX_MULSUB_2D 6, 4, 5, 17, _, _, 18, 19 ; t6a, t7a
523 ITX_MULSUB_2D 13, 11, 5, 17, _, _, 19, 18 ; t13, t12
524 ITX_MULSUB_2D 10, 7, 5, 17, _, _, 18, 19 ; t14, t15
525 REPX {psrad x, 14}, m8, m1, m14, m16
526 psubd m5, m8, m1 ; t10
527 paddd m1, m8 ; -out1
528 psubd m8, m15, m9 ; t3a
529 paddd m15, m9 ; -out15
530 psubd m9, m14, m16 ; t11
531 paddd m14, m16 ; out14
532 psubd m16, m0, m2 ; t2a
533 paddd m0, m2 ; out0
534 REPX {paddd x, m20}, m11, m13, m12, m3
535 paddd m2, m11, m10 ; out2
536 psubd m11, m10 ; t14a
537 psubd m10, m13, m7 ; t15a
538 paddd m13, m7 ; -out13
539 psubd m7, m12, m4 ; t7
540 paddd m12, m4 ; out12
541 psubd m4, m3, m6 ; t6
542 paddd m3, m6 ; -out3
543 REPX {psrad x, 14}, m10, m7, m11, m4
544 REPX {pmulld x, m21}, m9, m10, m7, m8, m5, m11, m4, m16
545 REPX {psrad x, 14}, m2, m13, m12, m3
546%endmacro
547
548%macro IADST16_PART2 0
549 paddd m9, m20
550 psubd m10, m20, m10
551 paddd m7, m20
552 psubd m8, m20, m8
553 paddd m6, m9, m5 ; out6
554 psubd m9, m5 ; out9
555 psubd m5, m10, m11 ; out5
556 paddd m10, m11 ; out10
557 psubd m11, m7, m4 ; out11
558 paddd m4, m7 ; out4
559 psubd m7, m8, m16 ; out7
560 paddd m8, m16 ; out8
561%endmacro
562
563%macro IADST16_PASS1_END 0
564 pxor m16, m16
565 psubd m1, m16, m1
566 psubd m3, m16, m3
567 psubd m13, m16, m13
568 psubd m15, m16, m15
569 REPX {psrad x, 14}, m4, m5, m6, m7, m8, m9, m10, m11
570%endmacro
571
572INV_TXFM_16X16_FN adst, dct, 39-18
573INV_TXFM_16X16_FN adst, adst
574
575cglobal vp9_iadst_16x16_internal_10, 0, 7, 22, dst, stride, c, eob, tx2
576 mova m0, [cq+64* 0]
577 mova m1, [cq+64* 1]
578 mova m2, [cq+64* 2]
579 mova m3, [cq+64* 3]
580 mova m4, [cq+64* 4]
581 mova m5, [cq+64* 5]
582 mova m6, [cq+64* 6]
583 mova m7, [cq+64* 7]
584 vpbroadcastd m20, [o(pd_8192)]
585 vpbroadcastd m21, [o(pd_11585)]
586 sub eobd, 39
587 jl .pass1_fast
588 mova m8, [cq+64* 8]
589 mova m9, [cq+64* 9]
590 mova m10, [cq+64*10]
591 mova m11, [cq+64*11]
592 mova m12, [cq+64*12]
593 mova m13, [cq+64*13]
594 mova m14, [cq+64*14]
595 mova m15, [cq+64*15]
596 call .main_part1
597 call .main_part2
598 IADST16_PASS1_END
599 jmp m(vp9_idct_16x16_internal_10).pass1_end
600.pass1_fast:
601 WRAP_YMM IADST16_PART1
602 WRAP_YMM IADST16_PART2
603 WRAP_YMM IADST16_PASS1_END
604 jmp m(vp9_idct_16x16_internal_10).pass1_fast_end
605.pass2:
606 test eobd, eobd
607 jl .pass2_fast
608 call .main_part1
609 jmp .pass2_end
610.pass2_fast:
611 call .main_part1_fast
612.pass2_end:
613 vpbroadcastd m20, [o(pd_532480)]
614 call .main_part2
615 vpbroadcastd m16, [o(pixel_clip6)]
616 REPX {paddd x, m16}, m0, m2, m12, m14
617 REPX {psubd x, m16, x}, m1, m3, m13, m15
618 REPX {psrad x, 6}, m0, m1, m2, m3
619 packssdw m0, m1
620 lea r6, [strideq*3]
621 packssdw m1, m2, m3
622 mova m2, [o(deintq_perm)]
623 vpbroadcastd m3, [o(pixel_clip)]
624 REPX {psrad x, 20}, m4, m5, m6, m7
625 call m(vp9_idct_16x16_internal_10).write_16x4
626 packssdw m0, m4, m5
627 packssdw m1, m6, m7
628 paddsw m0, m3
629 paddsw m1, m3
630 REPX {psrad x, 20}, m8, m9, m10, m11
631 call m(vp9_idct_16x16_internal_10).write_16x4
632 packssdw m0, m8, m9
633 packssdw m1, m10, m11
634 paddsw m0, m3
635 paddsw m1, m3
636 jmp m(vp9_idct_16x16_internal_10).pass2_end2
637ALIGN function_align
638 IADST16_PART1
639 ret
640ALIGN function_align
641.main_part2:
642 IADST16_PART2
643 ret
644
645cglobal vp9_idct_idct_32x32_add_10, 4, 7, 23, 64*64, dst, stride, c, eob
646%undef cmp
647 lea r5, [o_base]
648 dec eobd
649 jnz .pass1
650 imul r6d, [cq], 11585
651 vpbroadcastd m3, [o(pixel_clip)]
652 mov [cq], r3d
653 add r6d, 8192
654 sar r6d, 14
655 imul r6d, 11585
656 or r3d, 16
657 add r6d, 532480
658 sar r6d, 20
659 vpbroadcastw m2, r6d
660 paddsw m2, m3
661.dconly_loop:
662 paddsw m0, m2, [dstq+strideq*0]
663 paddsw m1, m2, [dstq+strideq*1]
664 psubusw m0, m3
665 psubusw m1, m3
666 mova [dstq+strideq*0], m0
667 mova [dstq+strideq*1], m1
668 lea dstq, [dstq+strideq*2]
669 dec r3d
670 jg .dconly_loop
671 RET
672.pass1:
673 vpbroadcastd m20, [o(pd_8192)]
674 vpbroadcastd m21, [o(pd_11585)]
675 cmp eobd, 135
676 jl .pass1_fast
677 add cq, 64
678 lea r4, [rsp+64*8]
679 cmp eobd, 579
680 jl .pass1_right_fast
681 mov r6d, 128*28
682 call .pass1_main
683 jmp .pass1_right_end
684.pass1_right_fast: ; bottomright quadrant is zero
685 mova m0, [cq+128* 1]
686 mova m1, [cq+128* 3]
687 mova m2, [cq+128* 5]
688 mova m3, [cq+128* 7]
689 mova m4, [cq+128* 9]
690 mova m5, [cq+128*11]
691 mova m6, [cq+128*13]
692 mova m7, [cq+128*15]
693 call .main_fast
694 mova m0, [cq+128* 0]
695 mova m1, [cq+128* 2]
696 mova m2, [cq+128* 4]
697 mova m3, [cq+128* 6]
698 mova m4, [cq+128* 8]
699 mova m5, [cq+128*10]
700 mova m6, [cq+128*12]
701 mova m7, [cq+128*14]
702 call m(vp9_idct_16x16_internal_10).main_part1_fast
703 mov r6d, 128*12
704 call .pass1_main_end
705.pass1_right_end:
706 mova [r4+64* 8], m0
707 mova [r4+64* 9], m1
708 mova [r4+64*10], m2
709 mova [r4+64*11], m3
710 mova [r4+64*12], m4
711 mova [r4+64*13], m5
712 mova [r4+64*14], m6
713 mova [r4+64*15], m7
714 mova [r4+64*16], m16
715 mova [r4+64*17], m17
716 mova [r4+64*18], m18
717 mova [r4+64*19], m19
718 mova [r4+64*20], m8
719 mova [r4+64*21], m9
720 mova [r4+64*22], m10
721 mova [r4+64*23], m11
722 sub cq, 64
723 sub r4, 64*8
724 mov r6d, 128*28
725 call .pass1_main
726 mova m12, [r4+64*20]
727 mova m13, [r4+64*21]
728 mova m14, [r4+64*22]
729 mova m15, [r4+64*23]
730 mova [r4+64*20], m8
731 mova [r4+64*21], m9
732 mova [r4+64*22], m10
733 mova [r4+64*23], m11
734 mova m8, [r4+64*16]
735 mova m9, [r4+64*17]
736 mova m10, [r4+64*18]
737 mova m11, [r4+64*19]
738 mova [r4+64*16], m16
739 mova [r4+64*17], m17
740 mova [r4+64*18], m18
741 mova [r4+64*19], m19
742 call .main
743 mova m0, [r4+64*16]
744 mova m1, [r4+64*17]
745 mova m2, [r4+64*18]
746 mova m3, [r4+64*19]
747 mova m4, [r4+64*20]
748 mova m5, [r4+64*21]
749 mova m6, [r4+64*22]
750 mova m7, [r4+64*23]
751 mova m8, [r4+64*24]
752 mova m9, [r4+64*25]
753 mova m10, [r4+64*26]
754 mova m11, [r4+64*27]
755 mova m12, [r4+64*28]
756 mova m13, [r4+64*29]
757 mova m14, [r4+64*30]
758 mova m15, [r4+64*31]
759 call m(vp9_idct_16x16_internal_10).main_part1
760 call .pass2_main_left
761 mova m8, [r4+64* 8]
762 mova m9, [r4+64* 9]
763 mova m10, [r4+64*10]
764 mova m11, [r4+64*11]
765 mova m12, [r4+64*12]
766 mova m13, [r4+64*13]
767 mova m14, [r4+64*14]
768 mova m15, [r4+64*15]
769 TRANSPOSE_4DQ 8, 10, 12, 14, 16
770 TRANSPOSE_4DQ 9, 11, 13, 15, 16
771 call .main
772 call .pass2_main_right
773 mova m8, [r4+64*24]
774 mova m9, [r4+64*25]
775 mova m10, [r4+64*26]
776 mova m11, [r4+64*27]
777 mova m12, [r4+64*28]
778 mova m13, [r4+64*29]
779 mova m14, [r4+64*30]
780 mova m15, [r4+64*31]
781 TRANSPOSE_4DQ 8, 10, 12, 14, 16
782 TRANSPOSE_4DQ 9, 11, 13, 15, 16
783 call m(vp9_idct_16x16_internal_10).main_part1
784 jmp .pass2_end
785.pass1_fast:
786 mova m0, [cq+128* 1]
787 mova m1, [cq+128* 3]
788 mova m2, [cq+128* 5]
789 mova m3, [cq+128* 7]
790 mova m4, [cq+128* 9]
791 mova m5, [cq+128*11]
792 mova m6, [cq+128*13]
793 mova m7, [cq+128*15]
794 mov r4, rsp
795 call .main_fast
796 mova m0, [cq+128* 0]
797 mova m1, [cq+128* 2]
798 mova m2, [cq+128* 4]
799 mova m3, [cq+128* 6]
800 mova m4, [cq+128* 8]
801 mova m5, [cq+128*10]
802 mova m6, [cq+128*12]
803 mova m7, [cq+128*14]
804 call m(vp9_idct_16x16_internal_10).main_part1_fast
805 call m(vp9_idct_16x16_internal_10).main_part2
806 mov r6d, 128*12
807 call .pass1_main_end2
808 mova [r4+64*16], m16
809 mova [r4+64*17], m17
810 mova [r4+64*18], m18
811 mova [r4+64*19], m19
812 mova [r4+64*20], m8
813 mova [r4+64*21], m9
814 mova [r4+64*22], m10
815 mova [r4+64*23], m11
816 call .main_fast
817 mova m0, [r4+64*16]
818 mova m1, [r4+64*17]
819 mova m2, [r4+64*18]
820 mova m3, [r4+64*19]
821 mova m4, [r4+64*20]
822 mova m5, [r4+64*21]
823 mova m6, [r4+64*22]
824 mova m7, [r4+64*23]
825 call m(vp9_idct_16x16_internal_10).main_part1_fast
826 call .pass2_main_left
827 call .main_fast
828 call .pass2_main_right
829 call m(vp9_idct_16x16_internal_10).main_part1_fast
830.pass2_end:
831 paddd m0, m22
832 paddd m18, m22
833 call m(vp9_idct_16x16_internal_10).main_part2
834 mova m20, [o(deintq_perm)]
835 rorx r2, strideq, 59 ; strideq*32
836 vpbroadcastd m21, [o(pixel_clip)]
837 add r2, dstq
838%assign i 0
839%rep 16
840 mova m16, [r4+64*(15-i)]
841 mova m17, [r4+64*(i-16)]
842 mova m18, [r4-64*(17+i)]
843 paddd m19, m %+ i, m16
844 psubd m0, m %+ i, m16
845 call .write_32x2
846 %assign i i+1
847%endrep
848 RET
849ALIGN function_align
850.write_32x2:
851 paddd m16, m17, m18
852 psubd m17, m18
853 REPX {psrad x, 6}, m19, m16, m0, m17
854 packssdw m16, m19
855 packssdw m17, m0
856 sub r2, strideq
857 vpermq m16, m20, m16
858 vpermq m17, m20, m17
859 paddsw m16, [dstq]
860 paddsw m17, [r2 ]
861 psubusw m16, m21
862 psubusw m17, m21
863 mova [dstq], m16
864 mova [r2 ], m17
865 add dstq, strideq
866 ret
867ALIGN function_align
868.pass1_main:
869 mova m0, [cq+128* 1]
870 mova m1, [cq+128* 3]
871 mova m2, [cq+128* 5]
872 mova m3, [cq+128* 7]
873 mova m4, [cq+128* 9]
874 mova m5, [cq+128*11]
875 mova m6, [cq+128*13]
876 mova m7, [cq+128*15]
877 mova m8, [cq+128*17]
878 mova m9, [cq+128*19]
879 mova m10, [cq+128*21]
880 mova m11, [cq+128*23]
881 mova m12, [cq+128*25]
882 mova m13, [cq+128*27]
883 mova m14, [cq+128*29]
884 mova m15, [cq+128*31]
885 call .main
886 mova m0, [cq+128* 0]
887 mova m1, [cq+128* 2]
888 mova m2, [cq+128* 4]
889 mova m3, [cq+128* 6]
890 mova m4, [cq+128* 8]
891 mova m5, [cq+128*10]
892 mova m6, [cq+128*12]
893 mova m7, [cq+128*14]
894 mova m8, [cq+128*16]
895 mova m9, [cq+128*18]
896 mova m10, [cq+128*20]
897 mova m11, [cq+128*22]
898 mova m12, [cq+128*24]
899 mova m13, [cq+128*26]
900 mova m14, [cq+128*28]
901 mova m15, [cq+128*30]
902 call m(vp9_idct_16x16_internal_10).main_part1
903.pass1_main_end:
904 call m(vp9_idct_16x16_internal_10).main_part2
905.pass1_main_end2:
906 pxor m16, m16
907.pass1_zero_loop:
908 mova [cq+r6+128*0], m16
909 mova [cq+r6+128*1], m16
910 mova [cq+r6+128*2], m16
911 mova [cq+r6+128*3], m16
912 sub r6d, 128*4
913 jge .pass1_zero_loop
914 mova m16, [r4+64*15]
915 mova m19, [r4+64*14]
916 mova m22, [r4+64*13]
917 mova m17, [r4+64*12]
918 psubd m18, m0, m16
919 paddd m16, m0
920 paddd m0, m19, m1
921 psubd m19, m1, m19
922 paddd m1, m17, m3
923 psubd m3, m17
924 paddd m17, m2, m22
925 psubd m2, m22
926 TRANSPOSE_4D 3, 2, 19, 18, 22 ; 28 29 30 31
927 TRANSPOSE_4D 16, 0, 17, 1, 22 ; 0 1 2 3
928 mova [r4+64*54], m3
929 mova [r4+64*55], m19
930 mova [r4+64*38], m2
931 mova [r4+64*39], m18
932 mova m2, [r4+64*11]
933 mova m19, [r4+64*10]
934 mova m3, [r4+64* 9]
935 mova m22, [r4+64* 8]
936 paddd m18, m4, m2
937 psubd m4, m2
938 paddd m2, m5, m19
939 psubd m5, m19
940 paddd m19, m6, m3
941 psubd m6, m3
942 paddd m3, m7, m22
943 psubd m7, m22
944 TRANSPOSE_4D 7, 6, 5, 4, 22 ; 24 25 26 27
945 TRANSPOSE_4D 18, 2, 19, 3, 22 ; 4 5 6 7
946 mova [r4+64*52], m7
947 mova [r4+64*53], m5
948 mova [r4+64*36], m6
949 mova [r4+64*37], m4
950 mova m7, [r4+64* 7]
951 mova m4, [r4+64* 6]
952 mova m5, [r4+64* 5]
953 mova m22, [r4+64* 4]
954 psubd m6, m8, m7
955 paddd m8, m7
956 psubd m7, m9, m4
957 paddd m4, m9
958 paddd m9, m10, m5
959 psubd m10, m5
960 paddd m5, m11, m22
961 psubd m11, m22
962 TRANSPOSE_4D 11, 10, 7, 6, 22 ; 20 21 22 23
963 TRANSPOSE_4D 8, 4, 9, 5, 22 ; 8 9 10 11
964 mova [r4+64*50], m11
965 mova [r4+64*51], m7
966 mova [r4+64*34], m10
967 mova [r4+64*35], m6
968 mova m6, [r4+64* 3]
969 mova m11, [r4+64* 2]
970 mova m7, [r4+64* 1]
971 mova m22, [r4+64* 0]
972 paddd m10, m12, m6
973 psubd m12, m6
974 paddd m6, m13, m11
975 psubd m13, m11
976 paddd m11, m14, m7
977 psubd m14, m7
978 paddd m7, m15, m22
979 psubd m15, m22
980 TRANSPOSE_4D 15, 14, 13, 12, 22 ; 16 17 18 19
981 TRANSPOSE_4D 10, 6, 11, 7, 22 ; 12 13 14 15
982 mova [r4+64*48], m15
983 mova [r4+64*49], m13
984 mova [r4+64*32], m14
985 mova [r4+64*33], m12
986 TRANSPOSE_4DQ 0, 2, 4, 6, 22
987 TRANSPOSE_4DQ 1, 3, 5, 7, 22
988 TRANSPOSE_4DQ 16, 18, 8, 10, 22
989 TRANSPOSE_4DQ 17, 19, 9, 11, 22
990 ret
991ALIGN function_align
992.pass2_main_left:
993 vpbroadcastd m22, [o(pixel_clip6)]
994 paddd m0, m22
995 paddd m18, m22
996 call m(vp9_idct_16x16_internal_10).main_part2
997 mova [r4+64*16], m0
998 mova [r4+64*17], m1
999 mova [r4+64*18], m2
1000 mova [r4+64*19], m3
1001 mova [r4+64*20], m4
1002 mova [r4+64*21], m5
1003 mova [r4+64*22], m6
1004 mova [r4+64*23], m7
1005 mova [r4+64*24], m8
1006 mova [r4+64*25], m9
1007 mova [r4+64*26], m10
1008 mova [r4+64*27], m11
1009 mova [r4+64*28], m12
1010 mova [r4+64*29], m13
1011 mova [r4+64*30], m14
1012 mova [r4+64*31], m15
1013 add r4, 64*32
1014 mova m0, [r4+64* 0]
1015 mova m1, [r4+64* 1]
1016 mova m2, [r4+64* 2]
1017 mova m3, [r4+64* 3]
1018 mova m4, [r4+64* 4]
1019 mova m5, [r4+64* 5]
1020 mova m6, [r4+64* 6]
1021 mova m7, [r4+64* 7]
1022 jmp .pass2_main_transpose
1023ALIGN function_align
1024.pass2_main_right:
1025 mova m0, [r4+64*16]
1026 mova m1, [r4+64*17]
1027 mova m2, [r4+64*18]
1028 mova m3, [r4+64*19]
1029 mova m4, [r4+64*20]
1030 mova m5, [r4+64*21]
1031 mova m6, [r4+64*22]
1032 mova m7, [r4+64*23]
1033.pass2_main_transpose:
1034 TRANSPOSE_4DQ 0, 2, 4, 6, 8
1035 TRANSPOSE_4DQ 1, 3, 5, 7, 8
1036 ret
1037ALIGN function_align
1038.main_fast:
1039 pmulld m15, m0, [o(pd_16364)] {1to16} ; t31a
1040 pmulld m0, [o(pd_804)] {1to16} ; t16a
1041 pmulld m8, m7, [o(pd_11003)] {1to16} ; t17a
1042 pmulld m7, [o(pd_12140)] {1to16} ; t30a
1043 pmulld m11, m4, [o(pd_14811)] {1to16} ; t29a
1044 pmulld m4, [o(pd_7005)] {1to16} ; t18a
1045 pmulld m12, m3, [o(pd_5520)] {1to16} ; t19a
1046 pmulld m3, [o(pd_15426)] {1to16} ; t28a
1047 pmulld m13, m2, [o(pd_15893)] {1to16} ; t27a
1048 pmulld m2, [o(pd_3981)] {1to16} ; t20a
1049 pmulld m10, m5, [o(pd_8423)] {1to16} ; t21a
1050 pmulld m5, [o(pd_14053)] {1to16} ; t26a
1051 pmulld m9, m6, [o(pd_13160)] {1to16} ; t25a
1052 pmulld m6, [o(pd_9760)] {1to16} ; t22a
1053 pmulld m14, m1, [o(pd_2404)] {1to16} ; t23a
1054 pmulld m1, [o(pd_16207)] {1to16} ; t24a
1055 REPX {psubd x, m20, x}, m8, m12, m10, m14
1056 jmp .main2
1057ALIGN function_align
1058.main:
1059 ITX_MULSUB_2D 0, 15, 16, 17, 18, _, 804, 16364 ; t16a, t31a
1060 ITX_MULSUB_2D 8, 7, 16, 17, 18, _, 12140, 11003 ; t17a, t30a
1061 ITX_MULSUB_2D 4, 11, 16, 17, 18, _, 7005, 14811 ; t18a, t29a
1062 ITX_MULSUB_2D 12, 3, 16, 17, 18, _, 15426, 5520 ; t19a, t28a
1063 ITX_MULSUB_2D 2, 13, 16, 17, 18, _, 3981, 15893 ; t20a, t27a
1064 ITX_MULSUB_2D 10, 5, 16, 17, 18, _, 14053, 8423 ; t21a, t26a
1065 ITX_MULSUB_2D 6, 9, 16, 17, 18, _, 9760, 13160 ; t22a, t25a
1066 ITX_MULSUB_2D 14, 1, 16, 17, 18, _, 16207, 2404 ; t23a, t24a
1067 REPX {paddd x, m20}, m8, m12, m10, m14
1068.main2:
1069 REPX {paddd x, m20}, m0, m15, m7, m4, m3, m11
1070 REPX {psrad x, 14 }, m8, m0, m15, m7, m12, m4, m3, m11
1071 psubd m16, m0, m8 ; t17
1072 paddd m0, m8 ; t16
1073 psubd m8, m15, m7 ; t30
1074 paddd m15, m7 ; t31
1075 paddd m7, m12, m4 ; t19
1076 psubd m12, m4 ; t18
1077 paddd m4, m3, m11 ; t28
1078 psubd m3, m11 ; t29
1079 REPX {paddd x, m20}, m2, m13, m5, m6, m1, m9
1080 REPX {psrad x, 14 }, m10, m2, m13, m5, m14, m6, m1, m9
1081 psubd m11, m2, m10 ; t21
1082 paddd m2, m10 ; t20
1083 psubd m10, m13, m5 ; t26
1084 paddd m13, m5 ; t27
1085 psubd m5, m14, m6 ; t22
1086 paddd m6, m14 ; t23
1087 psubd m14, m1, m9 ; t25
1088 paddd m9, m1 ; t24
1089 vpbroadcastd m19, [o(pd_16069)]
1090 vpbroadcastd m18, [o(pd_3196)]
1091 ITX_MULSUB_2D 8, 16, 1, 17, _, 20, 18, 19 ; t17a, t30a
1092 ITX_MULSUB_2D 3, 12, 1, 17, _, 20, 18, 19, 1 ; t29a, t18a
1093 vpbroadcastd m19, [o(pd_9102)]
1094 vpbroadcastd m18, [o(pd_13623)]
1095 ITX_MULSUB_2D 10, 11, 1, 17, _, 20, 18, 19 ; t21a, t26a
1096 ITX_MULSUB_2D 14, 5, 1, 17, _, 20, 18, 19, 1 ; t25a, t22a
1097 paddd m1, m6, m2 ; t23a
1098 psubd m6, m2 ; t20a
1099 psubd m2, m9, m13 ; t27a
1100 paddd m9, m13 ; t24a
1101 psubd m13, m15, m4 ; t28a
1102 paddd m15, m4 ; t31a
1103 psubd m4, m8, m12 ; t18
1104 paddd m8, m12 ; t17
1105 psubd m12, m0, m7 ; t19a
1106 paddd m0, m7 ; t16a
1107 psubd m7, m16, m3 ; t29
1108 paddd m3, m16 ; t30
1109 paddd m16, m5, m10 ; t22
1110 psubd m5, m10 ; t21
1111 psubd m10, m14, m11 ; t26
1112 paddd m14, m11 ; t25
1113 vpbroadcastd m19, [o(pd_15137)]
1114 vpbroadcastd m18, [o(pd_6270)]
1115 ITX_MULSUB_2D 13, 12, 11, 17, _, 20, 18, 19 ; t19, t28
1116 ITX_MULSUB_2D 2, 6, 11, 17, _, 20, 18, 19, 1 ; t27, t20
1117 ITX_MULSUB_2D 7, 4, 11, 17, _, 20, 18, 19 ; t18a, t29a
1118 ITX_MULSUB_2D 10, 5, 11, 17, _, 20, 18, 19, 1 ; t26a, t21a
1119 psubd m11, m0, m1 ; t23
1120 paddd m0, m1 ; t16
1121 paddd m1, m16, m8 ; t17a
1122 psubd m16, m8, m16 ; t22a
1123 psubd m8, m15, m9 ; t24
1124 paddd m15, m9 ; t31
1125 psubd m9, m3, m14 ; t25a
1126 paddd m14, m3 ; t30a
1127 paddd m3, m6, m13 ; t19a
1128 psubd m6, m13, m6 ; t20a
1129 paddd m13, m10, m4 ; t29
1130 psubd m10, m4, m10 ; t26
1131 psubd m4, m12, m2 ; t27a
1132 paddd m12, m2 ; t28a
1133 paddd m2, m7, m5 ; t18
1134 psubd m7, m5 ; t21
1135 REPX {pmulld x, m21}, m10, m8, m4, m9, m7, m11, m6, m16
1136 mova [r4+64* 0], m0
1137 mova [r4+64* 1], m1
1138 mova [r4+64* 2], m2
1139 mova [r4+64* 3], m3
1140 mova [r4+64*12], m12
1141 mova [r4+64*13], m13
1142 mova [r4+64*14], m14
1143 mova [r4+64*15], m15
1144 REPX {paddd x, m20}, m10, m8, m4, m9
1145 psubd m5, m10, m7 ; t21a
1146 paddd m10, m7 ; t26a
1147 psubd m7, m8, m11 ; t23a
1148 paddd m8, m11 ; t24a
1149 REPX {psrad x, 14 }, m5, m10, m7, m8
1150 paddd m11, m4, m6 ; t27
1151 psubd m4, m6 ; t20
1152 psubd m6, m9, m16 ; t22
1153 paddd m9, m16 ; t25
1154 REPX {psrad x, 14 }, m11, m4, m6, m9
1155 mova [r4+64* 4], m4
1156 mova [r4+64* 5], m5
1157 mova [r4+64* 6], m6
1158 mova [r4+64* 7], m7
1159 mova [r4+64* 8], m8
1160 mova [r4+64* 9], m9
1161 mova [r4+64*10], m10
1162 mova [r4+64*11], m11
1163 ret
1164
1165%endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t