avcodec/x86/vp9: Add AVX-512ICL for 16x16 and 32x32 10bpc inverse transforms

Henrik Gramner committed 1 year ago

Commit eda0ac71 parent df967d04 files · +1176 −2Browse files

Unified · Split

4 files

libavcodec/vp9dec.h +2 −2
libavcodec/x86/Makefile +1 −0
libavcodec/x86/vp9dsp_init_16bpp_template.c +8 −0
libavcodec/x86/vp9itxfm_16bpp_avx512.asm +1165 −0

@@ -220,8 +220,8 @@ struct VP9TileData { · 219 unmodified lines
220	DECLARE_ALIGNED(8, uint8_t, left_ref_ctx)[8];	220	DECLARE_ALIGNED(8, uint8_t, left_ref_ctx)[8];
221	DECLARE_ALIGNED(8, uint8_t, left_filter_ctx)[8];	221	DECLARE_ALIGNED(8, uint8_t, left_filter_ctx)[8];
222	// block reconstruction intermediates	222	// block reconstruction intermediates
223	DECLARE_ALIGNED(32, uint8_t, tmp_y)[64 * 64 * 2];	223	DECLARE_ALIGNED(64, uint8_t, tmp_y)[64 * 64 * 2];
224	DECLARE_ALIGNED(32, uint8_t, tmp_uv)[2][64 * 64 * 2];	224	DECLARE_ALIGNED(64, uint8_t, tmp_uv)[2][64 * 64 * 2];
225	struct { int x, y; } min_mv, max_mv;	225	struct { int x, y; } min_mv, max_mv;
226	int16_t block_base, block, uvblock_base[2], uvblock[2];	226	int16_t block_base, block, uvblock_base[2], uvblock[2];
227	uint8_t eob_base, uveob_base[2], eob, uveob[2];	227	uint8_t eob_base, uveob_base[2], eob, uveob[2];

libavcodec/x86/Makefile+1−0

@@ -186,6 +186,7 @@ X86ASM-OBJS-$(CONFIG_VP9_DECODER) += x86/vp9intrapred.o \ · 185 unmodified lines
186	x86/vp9itxfm.o \	186	x86/vp9itxfm.o \
187	x86/vp9itxfm_avx512.o \	187	x86/vp9itxfm_avx512.o \
188	x86/vp9itxfm_16bpp.o \	188	x86/vp9itxfm_16bpp.o \
		189	x86/vp9itxfm_16bpp_avx512.o \
189	x86/vp9lpf.o \	190	x86/vp9lpf.o \
190	x86/vp9lpf_16bpp.o \	191	x86/vp9lpf_16bpp.o \
191	x86/vp9mc.o \	192	x86/vp9mc.o \

libavcodec/x86/vp9dsp_init_16bpp_template.c+8−0

 #if BPC == 10
 decl_itxfm_func(idct,  idct,  4, BPC, mmxext);
 decl_itxfm_funcs(4, BPC, ssse3);
+decl_itxfm_funcs(16, BPC, avx512icl);
+decl_itxfm_func(idct,  idct, 32, BPC, avx512icl);
 #else
 decl_itxfm_func(idct,  idct,  4, BPC, sse2);
 #endif
 #endif
     }
+#if ARCH_X86_64 && BPC == 10
+    if (EXTERNAL_AVX512ICL(cpu_flags)) {
+        init_itx_funcs(TX_16X16, 16, BPC, avx512icl);
+        init_itx_func_one(TX_32X32, idct, idct, 32, BPC, avx512icl);
+    }
+#endif
 #endif /* HAVE_X86ASM */
     ff_vp9dsp_init_16bpp_x86(dsp);

libavcodec/x86/vp9itxfm_16bpp_avx512.asm+1165−0

@@ -0,0 +1,1165 @@
	1	;******************************************************************************
	2	;* VP9 IDCT SIMD optimizations
	3	;*
	4	;* Copyright (C) 2025 Two Orioles, LLC
	5	;*
	6	;* This file is part of FFmpeg.
	7	;*
	8	;* FFmpeg is free software; you can redistribute it and/or
	9	;* modify it under the terms of the GNU Lesser General Public
	10	;* License as published by the Free Software Foundation; either
	11	;* version 2.1 of the License, or (at your option) any later version.
	12	;*
	13	;* FFmpeg is distributed in the hope that it will be useful,
	14	;* but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	;* Lesser General Public License for more details.
	17	;*
	18	;* You should have received a copy of the GNU Lesser General Public
	19	;* License along with FFmpeg; if not, write to the Free Software
	20	;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	21	;******************************************************************************
	22
	23	%include "libavutil/x86/x86util.asm"
	24
	25	%if ARCH_X86_64 && HAVE_AVX512ICL_EXTERNAL
	26
	27	SECTION_RODATA 64
	28
	29	; Thw following set of constants are ordered to form the
	30	; qword shuffle mask { 0, 2, 4, 6, 1, 3, 5, 7 }
	31	%define deintq_perm pd_5520
	32	pd_5520: dd 5520
	33	pd_9760: dd 9760
	34	pd_10394: dd 10394
	35	pd_15426: dd 15426
	36	pd_804: dd 804
	37	pd_2404: dd 2404
	38	pd_6270: dd 6270
	39	pd_9102: dd 9102
	40	pd_11585: dd 11585
	41	pd_12665: dd 12665
	42	pd_7723: dd 7723
	43	pd_14811: dd 14811
	44	pd_7005: dd 7005
	45	pd_14053: dd 14053
	46	pd_8423: dd 8423
	47	pd_13623: dd 13623
	48
	49	pixel_clip: times 2 dw 0x7c00
	50	pixel_clip6: dd 2031648 ; 32 + (pixel_clip << 6)
	51	pd_532480: dd 532480 ; 8192 + (32 << 14)
	52	pd_8192: dd 8192
	53
	54	pd_1606: dd 1606
	55	pd_3196: dd 3196
	56	pd_3981: dd 3981
	57	pd_4756: dd 4756
	58	pd_11003: dd 11003
	59	pd_12140: dd 12140
	60	pd_13160: dd 13160
	61	pd_14449: dd 14449
	62	pd_15137: dd 15137
	63	pd_15679: dd 15679
	64	pd_15893: dd 15893
	65	pd_16069: dd 16069
	66	pd_16207: dd 16207
	67	pd_16305: dd 16305
	68	pd_16364: dd 16364
	69
	70	SECTION .text
	71
	72	%define o_base (deintq_perm+128)
	73	%define o(x) (r5 - o_base + (x))
	74	%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)
	75
	76	; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12
	77	; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12
	78	; skip round/shift if rnd is not a number
	79	%macro ITX_MULSUB_2D 8-9 0 ; dst/src[1-2], tmp[1-3], rnd, coef[1-2], inv_dst2
	80	%if %8 < 32
	81	pmulld m%4, m%1, m%8
	82	pmulld m%3, m%2, m%8
	83	%else
	84	vpbroadcastd m%3, [o(pd_%8)]
	85	pmulld m%4, m%1, m%3
	86	pmulld m%3, m%2
	87	%endif
	88	%if %7 < 32
	89	pmulld m%1, m%7
	90	pmulld m%2, m%7
	91	%else
	92	vpbroadcastd m%5, [o(pd_%7)]
	93	pmulld m%1, m%5
	94	pmulld m%2, m%5
	95	%endif
	96	%if %9
	97	psubd m%4, m%6, m%4
	98	psubd m%2, m%4, m%2
	99	%else
	100	%ifnum %6
	101	paddd m%4, m%6
	102	%endif
	103	paddd m%2, m%4
	104	%endif
	105	%ifnum %6
	106	paddd m%1, m%6
	107	%endif
	108	psubd m%1, m%3
	109	%ifnum %6
	110	psrad m%2, 14
	111	psrad m%1, 14
	112	%endif
	113	%endmacro
	114
	115	%macro WRAP_YMM 1+
	116	INIT_YMM cpuname
	117	%1
	118	INIT_ZMM cpuname
	119	%endmacro
	120
	121	%macro TRANSPOSE_4D 5 ; in[1-4], tmp
	122	punpckhdq m%5, m%3, m%4 ; c2 d2 c3 d3
	123	punpckldq m%3, m%4 ; c0 d0 c1 d1
	124	punpckhdq m%4, m%1, m%2 ; a2 b2 a3 b3
	125	punpckldq m%1, m%2 ; a0 b0 a1 b1
	126	punpckhqdq m%2, m%1, m%3 ; a1 b1 c1 d1
	127	punpcklqdq m%1, m%3 ; a0 b0 c0 d0
	128	punpcklqdq m%3, m%4, m%5 ; a2 b2 c2 d2
	129	punpckhqdq m%4, m%5 ; a3 b3 c3 d3
	130	%endmacro
	131
	132	%macro TRANSPOSE_4DQ 5 ; in[1-4], tmp
	133	vshufi32x4 m%5, m%3, m%4, q3232 ; c2 c3 d2 d3
	134	vinserti32x8 m%3, ym%4, 1 ; c0 c1 d0 d1
	135	vshufi32x4 m%4, m%1, m%2, q3232 ; a2 a3 b2 b3
	136	vinserti32x8 m%1, ym%2, 1 ; a0 a1 b0 b1
	137	vshufi32x4 m%2, m%1, m%3, q3131 ; a1 b1 c1 d1
	138	vshufi32x4 m%1, m%3, q2020 ; a0 b0 c0 d0
	139	vshufi32x4 m%3, m%4, m%5, q2020 ; a2 b2 c2 d2
	140	vshufi32x4 m%4, m%5, q3131 ; a3 b3 c3 d3
	141	%endmacro
	142
	143	%macro INV_TXFM_FN 3-4 0 ; type1, type2, size, eob_offset
	144	cglobal vp9_i%1_i%2_%3_add_10, 4, 5, 0, dst, stride, c, eob, tx2
	145	%define %%p1 m(vp9_i%1_%3_internal_10)
	146	lea r5, [o_base]
	147	; Jump to the 1st txfm function if we're not taking the fast path, which
	148	; in turn performs an indirect jump to the 2nd txfm function.
	149	lea tx2q, [m(vp9_i%2_%3_internal_10).pass2]
	150	%ifidn %1_%2, dct_dct
	151	dec eobd
	152	jnz %%p1
	153	%else
	154	%if %4
	155	add eobd, %4
	156	%endif
	157	; jump to the 1st txfm function unless it's located directly after this
	158	times ((%%end - %%p1) >> 31) & 1 jmp %%p1
	159	ALIGN function_align
	160	%%end:
	161	%endif
	162	%endmacro
	163
	164	%macro INV_TXFM_16X16_FN 2-3 0 ; type1, type2, eob_offset
	165	INV_TXFM_FN %1, %2, 16x16, %3
	166	%ifidn %1_%2, dct_dct
	167	imul r6d, [cq], 11585
	168	vpbroadcastd ym3, [o(pixel_clip)]
	169	mov [cq], r3d
	170	add r6d, 8192
	171	sar r6d, 14
	172	imul r6d, 11585
	173	or r3d, 8
	174	add r6d, 532480
	175	sar r6d, 20
	176	vpbroadcastw ym2, r6d
	177	paddsw ym2, ym3
	178	.dconly_loop:
	179	paddsw ym0, ym2, [dstq+strideq*0]
	180	paddsw ym1, ym2, [dstq+strideq*1]
	181	psubusw ym0, ym3
	182	psubusw ym1, ym3
	183	mova [dstq+strideq*0], ym0
	184	mova [dstq+strideq*1], ym1
	185	lea dstq, [dstq+strideq*2]
	186	dec r3d
	187	jg .dconly_loop
	188	RET
	189	%endif
	190	%endmacro
	191
	192	%macro IDCT16_PART1 0
	193	%if mmsize == 64
	194	.main_part1_fast:
	195	%endif
	196	pmulld m15, m1, [o(pd_16305)] {bcstd} ; t15a
	197	pmulld m1, [o(pd_1606)] {bcstd} ; t8a
	198	pmulld m9, m7, [o(pd_10394)] {bcstd} ; t9a
	199	pmulld m7, [o(pd_12665)] {bcstd} ; t14a
	200	pmulld m11, m5, [o(pd_14449)] {bcstd} ; t13a
	201	pmulld m5, [o(pd_7723)] {bcstd} ; t10a
	202	pmulld m13, m3, [o(pd_4756)] {bcstd} ; t11a
	203	pmulld m3, [o(pd_15679)] {bcstd} ; t12a
	204	pmulld m10, m6, [o(pd_9102)] {bcstd} ; t5a
	205	pmulld m6, [o(pd_13623)] {bcstd} ; t6a
	206	pmulld m14, m2, [o(pd_16069)] {bcstd} ; t7a
	207	pmulld m2, [o(pd_3196)] {bcstd} ; t4a
	208	pmulld m12, m4, [o(pd_15137)] {bcstd} ; t3
	209	pmulld m4, [o(pd_6270)] {bcstd} ; t2
	210	pmulld m0, m21
	211	REPX {psubd x, m20, x}, m9, m13, m10
	212	paddd m0, m20
	213	mova m18, m0
	214	%if mmsize == 64 ; for the ymm variant we only ever use the fast path
	215	jmp %%main_part1b
	216	.main_part1:
	217	ITX_MULSUB_2D 1, 15, 16, 17, 18, _, 1606, 16305 ; t8a, t15a
	218	ITX_MULSUB_2D 9, 7, 16, 17, 18, _, 12665, 10394 ; t9a, t14a
	219	ITX_MULSUB_2D 5, 11, 16, 17, 18, _, 7723, 14449 ; t10a, t13a
	220	ITX_MULSUB_2D 13, 3, 16, 17, 18, _, 15679, 4756 ; t11a, t12a
	221	ITX_MULSUB_2D 10, 6, 16, 17, 18, _, 13623, 9102 ; t5a, t6a
	222	ITX_MULSUB_2D 2, 14, 16, 17, 18, _, 3196, 16069 ; t4a, t7a
	223	ITX_MULSUB_2D 4, 12, 16, 17, 18, _, 6270, 15137 ; t2, t3
	224	pmulld m0, m21
	225	pmulld m8, m21
	226	REPX {paddd x, m20}, m0, m9, m13, m10
	227	psubd m18, m0, m8 ; t1
	228	paddd m0, m8 ; t0
	229	%%main_part1b:
	230	%endif
	231	vpbroadcastd m19, [o(pd_15137)]
	232	vpbroadcastd m16, [o(pd_6270)]
	233	REPX {paddd x, m20}, m15, m7, m1, m11, m3, m5
	234	REPX {psrad x, 14 }, m15, m7, m1, m9, m11, m3, m5, m13
	235	paddd m17, m15, m7 ; t15
	236	psubd m15, m7 ; t14
	237	psubd m7, m3, m11 ; t13
	238	paddd m3, m11 ; t12
	239	psubd m11, m13, m5 ; t10
	240	paddd m5, m13 ; t11
	241	psubd m13, m1, m9 ; t9
	242	paddd m1, m9 ; t8
	243	ITX_MULSUB_2D 15, 13, 8, 9, _, 20, 16, 19 ; t9a, t14a
	244	ITX_MULSUB_2D 7, 11, 8, 9, _, 20, 16, 19, 2 ; t13a, t10a
	245	paddd m16, m1, m5 ; t8a
	246	psubd m1, m5 ; t11a
	247	paddd m8, m15, m11 ; t9
	248	psubd m15, m11 ; t10
	249	psubd m11, m17, m3 ; t12a
	250	paddd m17, m3 ; t15a
	251	psubd m9, m13, m7 ; t13
	252	paddd m13, m7 ; t14
	253	REPX {pmulld x, m21}, m11, m9, m1, m15
	254	REPX {paddd x, m20}, m2, m6, m14
	255	REPX {psrad x, 14 }, m10, m2, m6, m14
	256	psubd m3, m2, m10 ; t5a
	257	paddd m10, m2 ; t4
	258	paddd m11, m20
	259	psubd m5, m11, m1 ; t11
	260	paddd m11, m1 ; t12
	261	psubd m1, m14, m6 ; t6a
	262	paddd m14, m6 ; t7
	263	pmulld m1, m21
	264	pmulld m3, m21
	265	paddd m4, m20
	266	paddd m12, m20
	267	REPX {psrad x, 14 }, m4, m12, m0, m18
	268	paddd m9, m20
	269	paddd m2, m9, m15 ; t13a
	270	psubd m9, m15 ; t10a
	271	paddd m1, m20
	272	psubd m6, m1, m3 ; t5
	273	paddd m1, m3 ; t6
	274	REPX {psrad x, 14}, m6, m1, m11, m5, m2, m9
	275	%endmacro
	276
	277	%macro IDCT16_PART2 0
	278	psubd m3, m0, m12 ; t3
	279	paddd m0, m12 ; t0
	280	psubd m12, m18, m4 ; t2
	281	paddd m18, m4 ; t1
	282	psubd m4, m3, m10 ; t4
	283	paddd m3, m10 ; t3
	284	psubd m10, m12, m6 ; t5
	285	paddd m12, m6 ; t2
	286	psubd m6, m18, m1 ; t6
	287	paddd m1, m18 ; t1
	288	psubd m7, m0, m14 ; t7
	289	paddd m0, m14 ; t0
	290	psubd m15, m0, m17 ; out15
	291	paddd m0, m17 ; out0
	292	psubd m14, m1, m13 ; out14
	293	paddd m1, m13 ; out1
	294	psubd m13, m12, m2 ; out13
	295	paddd m2, m12 ; out2
	296	psubd m12, m3, m11 ; out12
	297	paddd m3, m11 ; out3
	298	psubd m11, m4, m5 ; out11
	299	paddd m4, m5 ; out4
	300	paddd m5, m10, m9 ; out5
	301	psubd m10, m9 ; out10
	302	psubd m9, m6, m8 ; out9
	303	paddd m6, m8 ; out6
	304	psubd m8, m7, m16 ; out8
	305	paddd m7, m16 ; out7
	306	%endmacro
	307
	308	INIT_ZMM avx512icl
	309	INV_TXFM_16X16_FN dct, dct
	310	INV_TXFM_16X16_FN dct, adst, 39-23-1
	311
	312	cglobal vp9_idct_16x16_internal_10, 0, 7, 22, dst, stride, c, eob, tx2
	313	mova m0, [cq+64* 0]
	314	mova m1, [cq+64* 1]
	315	mova m2, [cq+64* 2]
	316	mova m3, [cq+64* 3]
	317	mova m4, [cq+64* 4]
	318	mova m5, [cq+64* 5]
	319	mova m6, [cq+64* 6]
	320	mova m7, [cq+64* 7]
	321	vpbroadcastd m20, [o(pd_8192)]
	322	vpbroadcastd m21, [o(pd_11585)]
	323	sub eobd, 38
	324	jl .pass1_fast
	325	mova m8, [cq+64* 8]
	326	mova m9, [cq+64* 9]
	327	mova m10, [cq+64*10]
	328	mova m11, [cq+64*11]
	329	mova m12, [cq+64*12]
	330	mova m13, [cq+64*13]
	331	mova m14, [cq+64*14]
	332	mova m15, [cq+64*15]
	333	call .main_part1
	334	call .main_part2
	335	.pass1_end:
	336	TRANSPOSE_4DQ 0, 4, 8, 12, 16
	337	TRANSPOSE_4DQ 1, 5, 9, 13, 16
	338	TRANSPOSE_4DQ 2, 6, 10, 14, 16
	339	TRANSPOSE_4DQ 3, 7, 11, 15, 16
	340	TRANSPOSE_4D 8, 9, 10, 11, 16
	341	TRANSPOSE_4D 12, 13, 14, 15, 16
	342	mov r6d, 64*12
	343	jmp .pass1_transpose_end
	344	.pass1_fast:
	345	WRAP_YMM IDCT16_PART1
	346	WRAP_YMM IDCT16_PART2
	347	.pass1_fast_end:
	348	vinserti32x8 m0, ym4, 1
	349	vinserti32x8 m8, ym12, 1
	350	vinserti32x8 m1, ym5, 1
	351	vinserti32x8 m9, ym13, 1
	352	vinserti32x8 m2, ym6, 1
	353	vinserti32x8 m10, ym14, 1
	354	vinserti32x8 m3, ym7, 1
	355	vinserti32x8 m11, ym15, 1
	356	vshufi32x4 m4, m0, m8, q3131
	357	vshufi32x4 m0, m8, q2020
	358	vshufi32x4 m5, m1, m9, q3131
	359	vshufi32x4 m1, m9, q2020
	360	vshufi32x4 m6, m2, m10, q3131
	361	vshufi32x4 m2, m10, q2020
	362	vshufi32x4 m7, m3, m11, q3131
	363	vshufi32x4 m3, m11, q2020
	364	mov r6d, 64*4
	365	.pass1_transpose_end:
	366	pxor m16, m16
	367	.zero_loop:
	368	mova [cq+r6+64*0], m16
	369	mova [cq+r6+64*1], m16
	370	mova [cq+r6+64*2], m16
	371	mova [cq+r6+64*3], m16
	372	sub r6d, 64*4
	373	jge .zero_loop
	374	TRANSPOSE_4D 0, 1, 2, 3, 16
	375	TRANSPOSE_4D 4, 5, 6, 7, 16
	376	jmp tx2q
	377	.pass2:
	378	test eobd, eobd
	379	jl .pass2_fast
	380	call .main_part1
	381	jmp .pass2_end
	382	.pass2_fast:
	383	call .main_part1_fast
	384	.pass2_end:
	385	vpbroadcastd m3, [o(pixel_clip6)]
	386	paddd m0, m3
	387	paddd m18, m3
	388	call .main_part2
	389	REPX {psrad x, 6}, m0, m1, m2, m3
	390	packssdw m0, m1
	391	lea r6, [strideq*3]
	392	packssdw m1, m2, m3
	393	mova m2, [o(deintq_perm)]
	394	vpbroadcastd m3, [o(pixel_clip)]
	395	REPX {psrad x, 6}, m4, m5, m6, m7
	396	call .write_16x4
	397	packssdw m0, m4, m5
	398	packssdw m1, m6, m7
	399	REPX {psrad x, 6}, m8, m9, m10, m11
	400	call .write_16x4
	401	packssdw m0, m8, m9
	402	packssdw m1, m10, m11
	403	.pass2_end2:
	404	REPX {psrad x, 6}, m12, m13, m14, m15
	405	call .write_16x4
	406	packssdw m0, m12, m13
	407	packssdw m1, m14, m15
	408	call .write_16x4
	409	RET
	410	ALIGN function_align
	411	.write_16x4:
	412	mova ym16, [dstq+strideq*0]
	413	vinserti32x8 m16, [dstq+strideq*1], 1
	414	mova ym17, [dstq+strideq*2]
	415	vinserti32x8 m17, [dstq+r6 ], 1
	416	vpermq m0, m2, m0
	417	vpermq m1, m2, m1
	418	paddsw m16, m0
	419	paddsw m17, m1
	420	psubusw m16, m3
	421	psubusw m17, m3
	422	mova [dstq+strideq*0], ym16
	423	vextracti32x8 [dstq+strideq*1], m16, 1
	424	mova [dstq+strideq*2], ym17
	425	vextracti32x8 [dstq+r6 ], m17, 1
	426	lea dstq, [dstq+strideq*4]
	427	ret
	428	ALIGN function_align
	429	IDCT16_PART1
	430	ret
	431	ALIGN function_align
	432	.main_part2:
	433	IDCT16_PART2
	434	ret
	435
	436	%macro IADST16_PART1 0
	437	%if mmsize == 64
	438	.main_part1_fast:
	439	%endif
	440	pmulld m15, m0, [o(pd_16364)] {bcstd} ; t1
	441	pmulld m0, [o(pd_804)] {bcstd} ; t0
	442	pmulld m13, m2, [o(pd_15893)] {bcstd} ; t3
	443	pmulld m2, [o(pd_3981)] {bcstd} ; t2
	444	pmulld m11, m4, [o(pd_14811)] {bcstd} ; t5
	445	pmulld m4, [o(pd_7005)] {bcstd} ; t4
	446	pmulld m9, m6, [o(pd_13160)] {bcstd} ; t7
	447	pmulld m6, [o(pd_9760)] {bcstd} ; t6
	448	pmulld m8, m7, [o(pd_11003)] {bcstd} ; t8
	449	pmulld m7, [o(pd_12140)] {bcstd} ; t9
	450	pmulld m10, m5, [o(pd_8423)] {bcstd} ; t10
	451	pmulld m5, [o(pd_14053)] {bcstd} ; t11
	452	pmulld m12, m3, [o(pd_5520)] {bcstd} ; t12
	453	pmulld m3, [o(pd_15426)] {bcstd} ; t13
	454	pmulld m14, m1, [o(pd_2404)] {bcstd} ; t14
	455	pmulld m1, [o(pd_16207)] {bcstd} ; t15
	456	REPX {psubd x, m20, x}, m15, m13, m11, m9
	457	%if mmsize == 64 ; for the ymm variant we only ever use the fast path
	458	jmp %%main_part1b
	459	ALIGN function_align
	460	.main_part1:
	461	ITX_MULSUB_2D 15, 0, 16, 17, 18, _, 804, 16364 ; t1, t0
	462	ITX_MULSUB_2D 13, 2, 16, 17, 18, _, 3981, 15893 ; t3, t2
	463	ITX_MULSUB_2D 11, 4, 16, 17, 18, _, 7005, 14811 ; t5, t4
	464	ITX_MULSUB_2D 9, 6, 16, 17, 18, _, 9760, 13160 ; t7, t6
	465	ITX_MULSUB_2D 7, 8, 16, 17, 18, _, 12140, 11003 ; t9, t8
	466	ITX_MULSUB_2D 5, 10, 16, 17, 18, _, 14053, 8423 ; t11, t10
	467	ITX_MULSUB_2D 3, 12, 16, 17, 18, _, 15426, 5520 ; t13, t12
	468	ITX_MULSUB_2D 1, 14, 16, 17, 18, _, 16207, 2404 ; t15, t14
	469	REPX {paddd x, m20}, m15, m13, m11, m9
	470	%%main_part1b:
	471	%endif
	472	REPX {paddd x, m20}, m0, m2, m4, m6
	473	psubd m16, m2, m10 ; t10a
	474	paddd m2, m10 ; t2a
	475	psubd m10, m9, m1 ; t15a
	476	paddd m9, m1 ; t7a
	477	psubd m1, m13, m5 ; t11a
	478	paddd m13, m5 ; t3a
	479	psubd m5, m6, m14 ; t14a
	480	paddd m6, m14 ; t6a
	481	REPX {psrad x, 14}, m16, m10, m1, m5
	482	psubd m14, m0, m8 ; t8a
	483	paddd m0, m8 ; t0a
	484	psubd m8, m15, m7 ; t9a
	485	paddd m15, m7 ; t1a
	486	psubd m7, m4, m12 ; t12a
	487	paddd m4, m12 ; t4a
	488	paddd m12, m11, m3 ; t5a
	489	psubd m11, m3 ; t13a
	490	REPX {psrad x, 14}, m14, m8, m7, m11
	491	vpbroadcastd m19, [o(pd_9102)]
	492	vpbroadcastd m18, [o(pd_13623)]
	493	ITX_MULSUB_2D 16, 1, 3, 17, _, _, 18, 19 ; t11, t10
	494	ITX_MULSUB_2D 10, 5, 3, 17, _, _, 19, 18 ; t14, t15
	495	vpbroadcastd m19, [o(pd_16069)]
	496	vpbroadcastd m18, [o(pd_3196)]
	497	ITX_MULSUB_2D 14, 8, 3, 17, _, _, 18, 19 ; t9, t8
	498	ITX_MULSUB_2D 11, 7, 3, 17, _, _, 19, 18 ; t12, t13
	499	vpbroadcastd m19, [o(pd_6270)]
	500	vpbroadcastd m18, [o(pd_15137)]
	501	REPX {psrad x, 14}, m15, m12, m0, m4
	502	psubd m3, m15, m12 ; t5
	503	paddd m15, m12 ; t1
	504	psubd m12, m0, m4 ; t4
	505	paddd m0, m4 ; t0
	506	REPX {psrad x, 14}, m2, m6, m13, m9
	507	psubd m4, m2, m6 ; t6
	508	paddd m2, m6 ; t2
	509	psubd m6, m13, m9 ; t7
	510	paddd m9, m13 ; t3
	511	REPX {paddd x, m20}, m8, m14, m1, m16
	512	psubd m13, m8, m11 ; t12a
	513	paddd m8, m11 ; t8a
	514	psubd m11, m14, m7 ; t13a
	515	paddd m14, m7 ; t9a
	516	psubd m7, m1, m10 ; t14a
	517	paddd m1, m10 ; t10a
	518	psubd m10, m16, m5 ; t15a
	519	paddd m16, m5 ; t11a
	520	REPX {psrad x, 14}, m13, m11, m7, m10
	521	ITX_MULSUB_2D 12, 3, 5, 17, _, _, 19, 18 ; t5a, t4a
	522	ITX_MULSUB_2D 6, 4, 5, 17, _, _, 18, 19 ; t6a, t7a
	523	ITX_MULSUB_2D 13, 11, 5, 17, _, _, 19, 18 ; t13, t12
	524	ITX_MULSUB_2D 10, 7, 5, 17, _, _, 18, 19 ; t14, t15
	525	REPX {psrad x, 14}, m8, m1, m14, m16
	526	psubd m5, m8, m1 ; t10
	527	paddd m1, m8 ; -out1
	528	psubd m8, m15, m9 ; t3a
	529	paddd m15, m9 ; -out15
	530	psubd m9, m14, m16 ; t11
	531	paddd m14, m16 ; out14
	532	psubd m16, m0, m2 ; t2a
	533	paddd m0, m2 ; out0
	534	REPX {paddd x, m20}, m11, m13, m12, m3
	535	paddd m2, m11, m10 ; out2
	536	psubd m11, m10 ; t14a
	537	psubd m10, m13, m7 ; t15a
	538	paddd m13, m7 ; -out13
	539	psubd m7, m12, m4 ; t7
	540	paddd m12, m4 ; out12
	541	psubd m4, m3, m6 ; t6
	542	paddd m3, m6 ; -out3
	543	REPX {psrad x, 14}, m10, m7, m11, m4
	544	REPX {pmulld x, m21}, m9, m10, m7, m8, m5, m11, m4, m16
	545	REPX {psrad x, 14}, m2, m13, m12, m3
	546	%endmacro
	547
	548	%macro IADST16_PART2 0
	549	paddd m9, m20
	550	psubd m10, m20, m10
	551	paddd m7, m20
	552	psubd m8, m20, m8
	553	paddd m6, m9, m5 ; out6
	554	psubd m9, m5 ; out9
	555	psubd m5, m10, m11 ; out5
	556	paddd m10, m11 ; out10
	557	psubd m11, m7, m4 ; out11
	558	paddd m4, m7 ; out4
	559	psubd m7, m8, m16 ; out7
	560	paddd m8, m16 ; out8
	561	%endmacro
	562
	563	%macro IADST16_PASS1_END 0
	564	pxor m16, m16
	565	psubd m1, m16, m1
	566	psubd m3, m16, m3
	567	psubd m13, m16, m13
	568	psubd m15, m16, m15
	569	REPX {psrad x, 14}, m4, m5, m6, m7, m8, m9, m10, m11
	570	%endmacro
	571
	572	INV_TXFM_16X16_FN adst, dct, 39-18
	573	INV_TXFM_16X16_FN adst, adst
	574
	575	cglobal vp9_iadst_16x16_internal_10, 0, 7, 22, dst, stride, c, eob, tx2
	576	mova m0, [cq+64* 0]
	577	mova m1, [cq+64* 1]
	578	mova m2, [cq+64* 2]
	579	mova m3, [cq+64* 3]
	580	mova m4, [cq+64* 4]
	581	mova m5, [cq+64* 5]
	582	mova m6, [cq+64* 6]
	583	mova m7, [cq+64* 7]
	584	vpbroadcastd m20, [o(pd_8192)]
	585	vpbroadcastd m21, [o(pd_11585)]
	586	sub eobd, 39
	587	jl .pass1_fast
	588	mova m8, [cq+64* 8]
	589	mova m9, [cq+64* 9]
	590	mova m10, [cq+64*10]
	591	mova m11, [cq+64*11]
	592	mova m12, [cq+64*12]
	593	mova m13, [cq+64*13]
	594	mova m14, [cq+64*14]
	595	mova m15, [cq+64*15]
	596	call .main_part1
	597	call .main_part2
	598	IADST16_PASS1_END
	599	jmp m(vp9_idct_16x16_internal_10).pass1_end
	600	.pass1_fast:
	601	WRAP_YMM IADST16_PART1
	602	WRAP_YMM IADST16_PART2
	603	WRAP_YMM IADST16_PASS1_END
	604	jmp m(vp9_idct_16x16_internal_10).pass1_fast_end
	605	.pass2:
	606	test eobd, eobd
	607	jl .pass2_fast
	608	call .main_part1
	609	jmp .pass2_end
	610	.pass2_fast:
	611	call .main_part1_fast
	612	.pass2_end:
	613	vpbroadcastd m20, [o(pd_532480)]
	614	call .main_part2
	615	vpbroadcastd m16, [o(pixel_clip6)]
	616	REPX {paddd x, m16}, m0, m2, m12, m14
	617	REPX {psubd x, m16, x}, m1, m3, m13, m15
	618	REPX {psrad x, 6}, m0, m1, m2, m3
	619	packssdw m0, m1
	620	lea r6, [strideq*3]
	621	packssdw m1, m2, m3
	622	mova m2, [o(deintq_perm)]
	623	vpbroadcastd m3, [o(pixel_clip)]
	624	REPX {psrad x, 20}, m4, m5, m6, m7
	625	call m(vp9_idct_16x16_internal_10).write_16x4
	626	packssdw m0, m4, m5
	627	packssdw m1, m6, m7
	628	paddsw m0, m3
	629	paddsw m1, m3
	630	REPX {psrad x, 20}, m8, m9, m10, m11
	631	call m(vp9_idct_16x16_internal_10).write_16x4
	632	packssdw m0, m8, m9
	633	packssdw m1, m10, m11
	634	paddsw m0, m3
	635	paddsw m1, m3
	636	jmp m(vp9_idct_16x16_internal_10).pass2_end2
	637	ALIGN function_align
	638	IADST16_PART1
	639	ret
	640	ALIGN function_align
	641	.main_part2:
	642	IADST16_PART2
	643	ret
	644
	645	cglobal vp9_idct_idct_32x32_add_10, 4, 7, 23, 64*64, dst, stride, c, eob
	646	%undef cmp
	647	lea r5, [o_base]
	648	dec eobd
	649	jnz .pass1
	650	imul r6d, [cq], 11585
	651	vpbroadcastd m3, [o(pixel_clip)]
	652	mov [cq], r3d
	653	add r6d, 8192
	654	sar r6d, 14
	655	imul r6d, 11585
	656	or r3d, 16
	657	add r6d, 532480
	658	sar r6d, 20
	659	vpbroadcastw m2, r6d
	660	paddsw m2, m3
	661	.dconly_loop:
	662	paddsw m0, m2, [dstq+strideq*0]
	663	paddsw m1, m2, [dstq+strideq*1]
	664	psubusw m0, m3
	665	psubusw m1, m3
	666	mova [dstq+strideq*0], m0
	667	mova [dstq+strideq*1], m1
	668	lea dstq, [dstq+strideq*2]
	669	dec r3d
	670	jg .dconly_loop
	671	RET
	672	.pass1:
	673	vpbroadcastd m20, [o(pd_8192)]
	674	vpbroadcastd m21, [o(pd_11585)]
	675	cmp eobd, 135
	676	jl .pass1_fast
	677	add cq, 64
	678	lea r4, [rsp+64*8]
	679	cmp eobd, 579
	680	jl .pass1_right_fast
	681	mov r6d, 128*28
	682	call .pass1_main
	683	jmp .pass1_right_end
	684	.pass1_right_fast: ; bottomright quadrant is zero
	685	mova m0, [cq+128* 1]
	686	mova m1, [cq+128* 3]
	687	mova m2, [cq+128* 5]
	688	mova m3, [cq+128* 7]
	689	mova m4, [cq+128* 9]
	690	mova m5, [cq+128*11]
	691	mova m6, [cq+128*13]
	692	mova m7, [cq+128*15]
	693	call .main_fast
	694	mova m0, [cq+128* 0]
	695	mova m1, [cq+128* 2]
	696	mova m2, [cq+128* 4]
	697	mova m3, [cq+128* 6]
	698	mova m4, [cq+128* 8]
	699	mova m5, [cq+128*10]
	700	mova m6, [cq+128*12]
	701	mova m7, [cq+128*14]
	702	call m(vp9_idct_16x16_internal_10).main_part1_fast
	703	mov r6d, 128*12
	704	call .pass1_main_end
	705	.pass1_right_end:
	706	mova [r4+64* 8], m0
	707	mova [r4+64* 9], m1
	708	mova [r4+64*10], m2
	709	mova [r4+64*11], m3
	710	mova [r4+64*12], m4
	711	mova [r4+64*13], m5
	712	mova [r4+64*14], m6
	713	mova [r4+64*15], m7
	714	mova [r4+64*16], m16
	715	mova [r4+64*17], m17
	716	mova [r4+64*18], m18
	717	mova [r4+64*19], m19
	718	mova [r4+64*20], m8
	719	mova [r4+64*21], m9
	720	mova [r4+64*22], m10
	721	mova [r4+64*23], m11
	722	sub cq, 64
	723	sub r4, 64*8
	724	mov r6d, 128*28
	725	call .pass1_main
	726	mova m12, [r4+64*20]
	727	mova m13, [r4+64*21]
	728	mova m14, [r4+64*22]
	729	mova m15, [r4+64*23]
	730	mova [r4+64*20], m8
	731	mova [r4+64*21], m9
	732	mova [r4+64*22], m10
	733	mova [r4+64*23], m11
	734	mova m8, [r4+64*16]
	735	mova m9, [r4+64*17]
	736	mova m10, [r4+64*18]
	737	mova m11, [r4+64*19]
	738	mova [r4+64*16], m16
	739	mova [r4+64*17], m17
	740	mova [r4+64*18], m18
	741	mova [r4+64*19], m19
	742	call .main
	743	mova m0, [r4+64*16]
	744	mova m1, [r4+64*17]
	745	mova m2, [r4+64*18]
	746	mova m3, [r4+64*19]
	747	mova m4, [r4+64*20]
	748	mova m5, [r4+64*21]
	749	mova m6, [r4+64*22]
	750	mova m7, [r4+64*23]
	751	mova m8, [r4+64*24]
	752	mova m9, [r4+64*25]
	753	mova m10, [r4+64*26]
	754	mova m11, [r4+64*27]
	755	mova m12, [r4+64*28]
	756	mova m13, [r4+64*29]
	757	mova m14, [r4+64*30]
	758	mova m15, [r4+64*31]
	759	call m(vp9_idct_16x16_internal_10).main_part1
	760	call .pass2_main_left
	761	mova m8, [r4+64* 8]
	762	mova m9, [r4+64* 9]
	763	mova m10, [r4+64*10]
	764	mova m11, [r4+64*11]
	765	mova m12, [r4+64*12]
	766	mova m13, [r4+64*13]
	767	mova m14, [r4+64*14]
	768	mova m15, [r4+64*15]
	769	TRANSPOSE_4DQ 8, 10, 12, 14, 16
	770	TRANSPOSE_4DQ 9, 11, 13, 15, 16
	771	call .main
	772	call .pass2_main_right
	773	mova m8, [r4+64*24]
	774	mova m9, [r4+64*25]
	775	mova m10, [r4+64*26]
	776	mova m11, [r4+64*27]
	777	mova m12, [r4+64*28]
	778	mova m13, [r4+64*29]
	779	mova m14, [r4+64*30]
	780	mova m15, [r4+64*31]
	781	TRANSPOSE_4DQ 8, 10, 12, 14, 16
	782	TRANSPOSE_4DQ 9, 11, 13, 15, 16
	783	call m(vp9_idct_16x16_internal_10).main_part1
	784	jmp .pass2_end
	785	.pass1_fast:
	786	mova m0, [cq+128* 1]
	787	mova m1, [cq+128* 3]
	788	mova m2, [cq+128* 5]
	789	mova m3, [cq+128* 7]
	790	mova m4, [cq+128* 9]
	791	mova m5, [cq+128*11]
	792	mova m6, [cq+128*13]
	793	mova m7, [cq+128*15]
	794	mov r4, rsp
	795	call .main_fast
	796	mova m0, [cq+128* 0]
	797	mova m1, [cq+128* 2]
	798	mova m2, [cq+128* 4]
	799	mova m3, [cq+128* 6]
	800	mova m4, [cq+128* 8]
	801	mova m5, [cq+128*10]
	802	mova m6, [cq+128*12]
	803	mova m7, [cq+128*14]
	804	call m(vp9_idct_16x16_internal_10).main_part1_fast
	805	call m(vp9_idct_16x16_internal_10).main_part2
	806	mov r6d, 128*12
	807	call .pass1_main_end2
	808	mova [r4+64*16], m16
	809	mova [r4+64*17], m17
	810	mova [r4+64*18], m18
	811	mova [r4+64*19], m19
	812	mova [r4+64*20], m8
	813	mova [r4+64*21], m9
	814	mova [r4+64*22], m10
	815	mova [r4+64*23], m11
	816	call .main_fast
	817	mova m0, [r4+64*16]
	818	mova m1, [r4+64*17]
	819	mova m2, [r4+64*18]
	820	mova m3, [r4+64*19]
	821	mova m4, [r4+64*20]
	822	mova m5, [r4+64*21]
	823	mova m6, [r4+64*22]
	824	mova m7, [r4+64*23]
	825	call m(vp9_idct_16x16_internal_10).main_part1_fast
	826	call .pass2_main_left
	827	call .main_fast
	828	call .pass2_main_right
	829	call m(vp9_idct_16x16_internal_10).main_part1_fast
	830	.pass2_end:
	831	paddd m0, m22
	832	paddd m18, m22
	833	call m(vp9_idct_16x16_internal_10).main_part2
	834	mova m20, [o(deintq_perm)]
	835	rorx r2, strideq, 59 ; strideq*32
	836	vpbroadcastd m21, [o(pixel_clip)]
	837	add r2, dstq
	838	%assign i 0
	839	%rep 16
	840	mova m16, [r4+64*(15-i)]
	841	mova m17, [r4+64*(i-16)]
	842	mova m18, [r4-64*(17+i)]
	843	paddd m19, m %+ i, m16
	844	psubd m0, m %+ i, m16
	845	call .write_32x2
	846	%assign i i+1
	847	%endrep
	848	RET
	849	ALIGN function_align
	850	.write_32x2:
	851	paddd m16, m17, m18
	852	psubd m17, m18
	853	REPX {psrad x, 6}, m19, m16, m0, m17
	854	packssdw m16, m19
	855	packssdw m17, m0
	856	sub r2, strideq
	857	vpermq m16, m20, m16
	858	vpermq m17, m20, m17
	859	paddsw m16, [dstq]
	860	paddsw m17, [r2 ]
	861	psubusw m16, m21
	862	psubusw m17, m21
	863	mova [dstq], m16
	864	mova [r2 ], m17
	865	add dstq, strideq
	866	ret
	867	ALIGN function_align
	868	.pass1_main:
	869	mova m0, [cq+128* 1]
	870	mova m1, [cq+128* 3]
	871	mova m2, [cq+128* 5]
	872	mova m3, [cq+128* 7]
	873	mova m4, [cq+128* 9]
	874	mova m5, [cq+128*11]
	875	mova m6, [cq+128*13]
	876	mova m7, [cq+128*15]
	877	mova m8, [cq+128*17]
	878	mova m9, [cq+128*19]
	879	mova m10, [cq+128*21]
	880	mova m11, [cq+128*23]
	881	mova m12, [cq+128*25]
	882	mova m13, [cq+128*27]
	883	mova m14, [cq+128*29]
	884	mova m15, [cq+128*31]
	885	call .main
	886	mova m0, [cq+128* 0]
	887	mova m1, [cq+128* 2]
	888	mova m2, [cq+128* 4]
	889	mova m3, [cq+128* 6]
	890	mova m4, [cq+128* 8]
	891	mova m5, [cq+128*10]
	892	mova m6, [cq+128*12]
	893	mova m7, [cq+128*14]
	894	mova m8, [cq+128*16]
	895	mova m9, [cq+128*18]
	896	mova m10, [cq+128*20]
	897	mova m11, [cq+128*22]
	898	mova m12, [cq+128*24]
	899	mova m13, [cq+128*26]
	900	mova m14, [cq+128*28]
	901	mova m15, [cq+128*30]
	902	call m(vp9_idct_16x16_internal_10).main_part1
	903	.pass1_main_end:
	904	call m(vp9_idct_16x16_internal_10).main_part2
	905	.pass1_main_end2:
	906	pxor m16, m16
	907	.pass1_zero_loop:
	908	mova [cq+r6+128*0], m16
	909	mova [cq+r6+128*1], m16
	910	mova [cq+r6+128*2], m16
	911	mova [cq+r6+128*3], m16
	912	sub r6d, 128*4
	913	jge .pass1_zero_loop
	914	mova m16, [r4+64*15]
	915	mova m19, [r4+64*14]
	916	mova m22, [r4+64*13]
	917	mova m17, [r4+64*12]
	918	psubd m18, m0, m16
	919	paddd m16, m0
	920	paddd m0, m19, m1
	921	psubd m19, m1, m19
	922	paddd m1, m17, m3
	923	psubd m3, m17
	924	paddd m17, m2, m22
	925	psubd m2, m22
	926	TRANSPOSE_4D 3, 2, 19, 18, 22 ; 28 29 30 31
	927	TRANSPOSE_4D 16, 0, 17, 1, 22 ; 0 1 2 3
	928	mova [r4+64*54], m3
	929	mova [r4+64*55], m19
	930	mova [r4+64*38], m2
	931	mova [r4+64*39], m18
	932	mova m2, [r4+64*11]
	933	mova m19, [r4+64*10]
	934	mova m3, [r4+64* 9]
	935	mova m22, [r4+64* 8]
	936	paddd m18, m4, m2
	937	psubd m4, m2
	938	paddd m2, m5, m19
	939	psubd m5, m19
	940	paddd m19, m6, m3
	941	psubd m6, m3
	942	paddd m3, m7, m22
	943	psubd m7, m22
	944	TRANSPOSE_4D 7, 6, 5, 4, 22 ; 24 25 26 27
	945	TRANSPOSE_4D 18, 2, 19, 3, 22 ; 4 5 6 7
	946	mova [r4+64*52], m7
	947	mova [r4+64*53], m5
	948	mova [r4+64*36], m6
	949	mova [r4+64*37], m4
	950	mova m7, [r4+64* 7]
	951	mova m4, [r4+64* 6]
	952	mova m5, [r4+64* 5]
	953	mova m22, [r4+64* 4]
	954	psubd m6, m8, m7
	955	paddd m8, m7
	956	psubd m7, m9, m4
	957	paddd m4, m9
	958	paddd m9, m10, m5
	959	psubd m10, m5
	960	paddd m5, m11, m22
	961	psubd m11, m22
	962	TRANSPOSE_4D 11, 10, 7, 6, 22 ; 20 21 22 23
	963	TRANSPOSE_4D 8, 4, 9, 5, 22 ; 8 9 10 11
	964	mova [r4+64*50], m11
	965	mova [r4+64*51], m7
	966	mova [r4+64*34], m10
	967	mova [r4+64*35], m6
	968	mova m6, [r4+64* 3]
	969	mova m11, [r4+64* 2]
	970	mova m7, [r4+64* 1]
	971	mova m22, [r4+64* 0]
	972	paddd m10, m12, m6
	973	psubd m12, m6
	974	paddd m6, m13, m11
	975	psubd m13, m11
	976	paddd m11, m14, m7
	977	psubd m14, m7
	978	paddd m7, m15, m22
	979	psubd m15, m22
	980	TRANSPOSE_4D 15, 14, 13, 12, 22 ; 16 17 18 19
	981	TRANSPOSE_4D 10, 6, 11, 7, 22 ; 12 13 14 15
	982	mova [r4+64*48], m15
	983	mova [r4+64*49], m13
	984	mova [r4+64*32], m14
	985	mova [r4+64*33], m12
	986	TRANSPOSE_4DQ 0, 2, 4, 6, 22
	987	TRANSPOSE_4DQ 1, 3, 5, 7, 22
	988	TRANSPOSE_4DQ 16, 18, 8, 10, 22
	989	TRANSPOSE_4DQ 17, 19, 9, 11, 22
	990	ret
	991	ALIGN function_align
	992	.pass2_main_left:
	993	vpbroadcastd m22, [o(pixel_clip6)]
	994	paddd m0, m22
	995	paddd m18, m22
	996	call m(vp9_idct_16x16_internal_10).main_part2
	997	mova [r4+64*16], m0
	998	mova [r4+64*17], m1
	999	mova [r4+64*18], m2
	1000	mova [r4+64*19], m3
	1001	mova [r4+64*20], m4
	1002	mova [r4+64*21], m5
	1003	mova [r4+64*22], m6
	1004	mova [r4+64*23], m7
	1005	mova [r4+64*24], m8
	1006	mova [r4+64*25], m9
	1007	mova [r4+64*26], m10
	1008	mova [r4+64*27], m11
	1009	mova [r4+64*28], m12
	1010	mova [r4+64*29], m13
	1011	mova [r4+64*30], m14
	1012	mova [r4+64*31], m15
	1013	add r4, 64*32
	1014	mova m0, [r4+64* 0]
	1015	mova m1, [r4+64* 1]
	1016	mova m2, [r4+64* 2]
	1017	mova m3, [r4+64* 3]
	1018	mova m4, [r4+64* 4]
	1019	mova m5, [r4+64* 5]
	1020	mova m6, [r4+64* 6]
	1021	mova m7, [r4+64* 7]
	1022	jmp .pass2_main_transpose
	1023	ALIGN function_align
	1024	.pass2_main_right:
	1025	mova m0, [r4+64*16]
	1026	mova m1, [r4+64*17]
	1027	mova m2, [r4+64*18]
	1028	mova m3, [r4+64*19]
	1029	mova m4, [r4+64*20]
	1030	mova m5, [r4+64*21]
	1031	mova m6, [r4+64*22]
	1032	mova m7, [r4+64*23]
	1033	.pass2_main_transpose:
	1034	TRANSPOSE_4DQ 0, 2, 4, 6, 8
	1035	TRANSPOSE_4DQ 1, 3, 5, 7, 8
	1036	ret
	1037	ALIGN function_align
	1038	.main_fast:
	1039	pmulld m15, m0, [o(pd_16364)] {1to16} ; t31a
	1040	pmulld m0, [o(pd_804)] {1to16} ; t16a
	1041	pmulld m8, m7, [o(pd_11003)] {1to16} ; t17a
	1042	pmulld m7, [o(pd_12140)] {1to16} ; t30a
	1043	pmulld m11, m4, [o(pd_14811)] {1to16} ; t29a
	1044	pmulld m4, [o(pd_7005)] {1to16} ; t18a
	1045	pmulld m12, m3, [o(pd_5520)] {1to16} ; t19a
	1046	pmulld m3, [o(pd_15426)] {1to16} ; t28a
	1047	pmulld m13, m2, [o(pd_15893)] {1to16} ; t27a
	1048	pmulld m2, [o(pd_3981)] {1to16} ; t20a
	1049	pmulld m10, m5, [o(pd_8423)] {1to16} ; t21a
	1050	pmulld m5, [o(pd_14053)] {1to16} ; t26a
	1051	pmulld m9, m6, [o(pd_13160)] {1to16} ; t25a
	1052	pmulld m6, [o(pd_9760)] {1to16} ; t22a
	1053	pmulld m14, m1, [o(pd_2404)] {1to16} ; t23a
	1054	pmulld m1, [o(pd_16207)] {1to16} ; t24a
	1055	REPX {psubd x, m20, x}, m8, m12, m10, m14
	1056	jmp .main2
	1057	ALIGN function_align
	1058	.main:
	1059	ITX_MULSUB_2D 0, 15, 16, 17, 18, _, 804, 16364 ; t16a, t31a
	1060	ITX_MULSUB_2D 8, 7, 16, 17, 18, _, 12140, 11003 ; t17a, t30a
	1061	ITX_MULSUB_2D 4, 11, 16, 17, 18, _, 7005, 14811 ; t18a, t29a
	1062	ITX_MULSUB_2D 12, 3, 16, 17, 18, _, 15426, 5520 ; t19a, t28a
	1063	ITX_MULSUB_2D 2, 13, 16, 17, 18, _, 3981, 15893 ; t20a, t27a
	1064	ITX_MULSUB_2D 10, 5, 16, 17, 18, _, 14053, 8423 ; t21a, t26a
	1065	ITX_MULSUB_2D 6, 9, 16, 17, 18, _, 9760, 13160 ; t22a, t25a
	1066	ITX_MULSUB_2D 14, 1, 16, 17, 18, _, 16207, 2404 ; t23a, t24a
	1067	REPX {paddd x, m20}, m8, m12, m10, m14
	1068	.main2:
	1069	REPX {paddd x, m20}, m0, m15, m7, m4, m3, m11
	1070	REPX {psrad x, 14 }, m8, m0, m15, m7, m12, m4, m3, m11
	1071	psubd m16, m0, m8 ; t17
	1072	paddd m0, m8 ; t16
	1073	psubd m8, m15, m7 ; t30
	1074	paddd m15, m7 ; t31
	1075	paddd m7, m12, m4 ; t19
	1076	psubd m12, m4 ; t18
	1077	paddd m4, m3, m11 ; t28
	1078	psubd m3, m11 ; t29
	1079	REPX {paddd x, m20}, m2, m13, m5, m6, m1, m9
	1080	REPX {psrad x, 14 }, m10, m2, m13, m5, m14, m6, m1, m9
	1081	psubd m11, m2, m10 ; t21
	1082	paddd m2, m10 ; t20
	1083	psubd m10, m13, m5 ; t26
	1084	paddd m13, m5 ; t27
	1085	psubd m5, m14, m6 ; t22
	1086	paddd m6, m14 ; t23
	1087	psubd m14, m1, m9 ; t25
	1088	paddd m9, m1 ; t24
	1089	vpbroadcastd m19, [o(pd_16069)]
	1090	vpbroadcastd m18, [o(pd_3196)]
	1091	ITX_MULSUB_2D 8, 16, 1, 17, _, 20, 18, 19 ; t17a, t30a
	1092	ITX_MULSUB_2D 3, 12, 1, 17, _, 20, 18, 19, 1 ; t29a, t18a
	1093	vpbroadcastd m19, [o(pd_9102)]
	1094	vpbroadcastd m18, [o(pd_13623)]
	1095	ITX_MULSUB_2D 10, 11, 1, 17, _, 20, 18, 19 ; t21a, t26a
	1096	ITX_MULSUB_2D 14, 5, 1, 17, _, 20, 18, 19, 1 ; t25a, t22a
	1097	paddd m1, m6, m2 ; t23a
	1098	psubd m6, m2 ; t20a
	1099	psubd m2, m9, m13 ; t27a
	1100	paddd m9, m13 ; t24a
	1101	psubd m13, m15, m4 ; t28a
	1102	paddd m15, m4 ; t31a
	1103	psubd m4, m8, m12 ; t18
	1104	paddd m8, m12 ; t17
	1105	psubd m12, m0, m7 ; t19a
	1106	paddd m0, m7 ; t16a
	1107	psubd m7, m16, m3 ; t29
	1108	paddd m3, m16 ; t30
	1109	paddd m16, m5, m10 ; t22
	1110	psubd m5, m10 ; t21
	1111	psubd m10, m14, m11 ; t26
	1112	paddd m14, m11 ; t25
	1113	vpbroadcastd m19, [o(pd_15137)]
	1114	vpbroadcastd m18, [o(pd_6270)]
	1115	ITX_MULSUB_2D 13, 12, 11, 17, _, 20, 18, 19 ; t19, t28
	1116	ITX_MULSUB_2D 2, 6, 11, 17, _, 20, 18, 19, 1 ; t27, t20
	1117	ITX_MULSUB_2D 7, 4, 11, 17, _, 20, 18, 19 ; t18a, t29a
	1118	ITX_MULSUB_2D 10, 5, 11, 17, _, 20, 18, 19, 1 ; t26a, t21a
	1119	psubd m11, m0, m1 ; t23
	1120	paddd m0, m1 ; t16
	1121	paddd m1, m16, m8 ; t17a
	1122	psubd m16, m8, m16 ; t22a
	1123	psubd m8, m15, m9 ; t24
	1124	paddd m15, m9 ; t31
	1125	psubd m9, m3, m14 ; t25a
	1126	paddd m14, m3 ; t30a
	1127	paddd m3, m6, m13 ; t19a
	1128	psubd m6, m13, m6 ; t20a
	1129	paddd m13, m10, m4 ; t29
	1130	psubd m10, m4, m10 ; t26
	1131	psubd m4, m12, m2 ; t27a
	1132	paddd m12, m2 ; t28a
	1133	paddd m2, m7, m5 ; t18
	1134	psubd m7, m5 ; t21
	1135	REPX {pmulld x, m21}, m10, m8, m4, m9, m7, m11, m6, m16
	1136	mova [r4+64* 0], m0
	1137	mova [r4+64* 1], m1
	1138	mova [r4+64* 2], m2
	1139	mova [r4+64* 3], m3
	1140	mova [r4+64*12], m12
	1141	mova [r4+64*13], m13
	1142	mova [r4+64*14], m14
	1143	mova [r4+64*15], m15
	1144	REPX {paddd x, m20}, m10, m8, m4, m9
	1145	psubd m5, m10, m7 ; t21a
	1146	paddd m10, m7 ; t26a
	1147	psubd m7, m8, m11 ; t23a
	1148	paddd m8, m11 ; t24a
	1149	REPX {psrad x, 14 }, m5, m10, m7, m8
	1150	paddd m11, m4, m6 ; t27
	1151	psubd m4, m6 ; t20
	1152	psubd m6, m9, m16 ; t22
	1153	paddd m9, m16 ; t25
	1154	REPX {psrad x, 14 }, m11, m4, m6, m9
	1155	mova [r4+64* 4], m4
	1156	mova [r4+64* 5], m5
	1157	mova [r4+64* 6], m6
	1158	mova [r4+64* 7], m7
	1159	mova [r4+64* 8], m8
	1160	mova [r4+64* 9], m9
	1161	mova [r4+64*10], m10
	1162	mova [r4+64*11], m11
	1163	ret
	1164
	1165	%endif

avcodec/x86/vp9: Add AVX-512ICL for 16x16 and 32x32 10bpc inverse transforms

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code