avcodec/x86/vp9: Add AVX-512ICL for 16x16 and 32x32 8bpc inverse transforms

Henrik Gramner committed 1 year ago

Commit fd18ae81 parent b6803bf5 files · +1654 −7Browse files

Unified · Split

5 files

libavcodec/x86/Makefile +1 −0
libavcodec/x86/vp9dsp_init.c +15 −0
libavcodec/x86/vp9itxfm_avx512.asm +1629 −0
libavutil/mem_internal.h +2 −0
tests/checkasm/vp9dsp.c +7 −7

@@ -184,6 +184,7 @@ X86ASM-OBJS-$(CONFIG_VP6_DECODER) += x86/vp6dsp.o · 183 unmodified lines
184	X86ASM-OBJS-$(CONFIG_VP9_DECODER) += x86/vp9intrapred.o \	184	X86ASM-OBJS-$(CONFIG_VP9_DECODER) += x86/vp9intrapred.o \
185	x86/vp9intrapred_16bpp.o \	185	x86/vp9intrapred_16bpp.o \
186	x86/vp9itxfm.o \	186	x86/vp9itxfm.o \
		187	x86/vp9itxfm_avx512.o \
187	x86/vp9itxfm_16bpp.o \	188	x86/vp9itxfm_16bpp.o \
188	x86/vp9lpf.o \	189	x86/vp9lpf.o \
189	x86/vp9lpf_16bpp.o \	190	x86/vp9lpf_16bpp.o \

libavcodec/x86/vp9dsp_init.c+15−0

@@ -114,7 +114,9 @@ itxfm_func(idct, idct, 32, ssse3); · 113 unmodified lines
114	itxfm_func(idct, idct, 32, avx);	114	itxfm_func(idct, idct, 32, avx);
115	itxfm_func(iwht, iwht, 4, mmx);	115	itxfm_func(iwht, iwht, 4, mmx);
116	itxfm_funcs(16, avx2);	116	itxfm_funcs(16, avx2);
		117	itxfm_funcs(16, avx512icl);
117	itxfm_func(idct, idct, 32, avx2);	118	itxfm_func(idct, idct, 32, avx2);
		119	itxfm_func(idct, idct, 32, avx512icl);
118		120
119	#undef itxfm_func	121	#undef itxfm_func
120	#undef itxfm_funcs	122	#undef itxfm_funcs
@@ -406,6 +408,19 @@ av_cold void ff_vp9dsp_init_x86(VP9DSPContext *dsp, int bpp, int bitexact) · 285 unmodified lines
406	init_ipred(32, avx2, tm, TM_VP8);	408	init_ipred(32, avx2, tm, TM_VP8);
407	}	409	}
408		410
		411	#if ARCH_X86_64
		412	if (EXTERNAL_AVX512ICL(cpu_flags)) {
		413	dsp->itxfm_add[TX_16X16][DCT_DCT] = ff_vp9_idct_idct_16x16_add_avx512icl;
		414	dsp->itxfm_add[TX_16X16][ADST_DCT] = ff_vp9_idct_iadst_16x16_add_avx512icl;
		415	dsp->itxfm_add[TX_16X16][DCT_ADST] = ff_vp9_iadst_idct_16x16_add_avx512icl;
		416	dsp->itxfm_add[TX_16X16][ADST_ADST] = ff_vp9_iadst_iadst_16x16_add_avx512icl;
		417	dsp->itxfm_add[TX_32X32][ADST_ADST] =
		418	dsp->itxfm_add[TX_32X32][ADST_DCT] =
		419	dsp->itxfm_add[TX_32X32][DCT_ADST] =
		420	dsp->itxfm_add[TX_32X32][DCT_DCT] = ff_vp9_idct_idct_32x32_add_avx512icl;
		421	}
		422	#endif
		423
409	#undef init_fpel	424	#undef init_fpel
410	#undef init_subpel1	425	#undef init_subpel1
411	#undef init_subpel2	426	#undef init_subpel2

libavcodec/x86/vp9itxfm_avx512.asm+1629−0

@@ -0,0 +1,1629 @@
	1	;******************************************************************************
	2	;* VP9 IDCT SIMD optimizations
	3	;*
	4	;* Copyright (C) 2025 Two Orioles, LLC
	5	;*
	6	;* This file is part of FFmpeg.
	7	;*
	8	;* FFmpeg is free software; you can redistribute it and/or
	9	;* modify it under the terms of the GNU Lesser General Public
	10	;* License as published by the Free Software Foundation; either
	11	;* version 2.1 of the License, or (at your option) any later version.
	12	;*
	13	;* FFmpeg is distributed in the hope that it will be useful,
	14	;* but WITHOUT ANY WARRANTY; without even the implied warranty of
	15	;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	16	;* Lesser General Public License for more details.
	17	;*
	18	;* You should have received a copy of the GNU Lesser General Public
	19	;* License along with FFmpeg; if not, write to the Free Software
	20	;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	21	;******************************************************************************
	22
	23	%include "libavutil/x86/x86util.asm"
	24
	25	%if ARCH_X86_64 && HAVE_AVX512ICL_EXTERNAL
	26
	27	SECTION_RODATA 64
	28
	29	dup16_perm: db 0, 1, 0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7, 6, 7
	30	db 8, 9, 8, 9, 10, 11, 10, 11, 12, 13, 12, 13, 14, 15, 14, 15
	31	db 16, 17, 16, 17, 18, 19, 18, 19, 20, 21, 20, 21, 22, 23, 22, 23
	32	db 24, 25, 24, 25, 26, 27, 26, 27, 28, 29, 28, 29, 30, 31, 30, 31
	33	itx_perm: dq 0x0000000820150440, 0x0000000231372604
	34	dq 0x0000000ca8041551, 0x00000006b9263715
	35	dq 0x00000001ec9d8c62, 0x0000000bfdbfae26
	36	dq 0x00000005648c9d73, 0x0000000f75aebf37
	37	deint_shuf: db 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15
	38	int_shuf1: db 0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15
	39	int_shuf2: db 8, 9, 0, 1, 10, 11, 2, 3, 12, 13, 4, 5, 14, 15, 6, 7
	40	pw_512: times 4 dw 512
	41	pw_m512: times 4 dw -512
	42	pw_15137_6270x2x4: times 4 dw 15137*2
	43	times 4 dw 6270*2
	44	pw_11585_m11585x2x4: times 4 dw 11585*2
	45	pw_m11585_11585x2x4: times 4 dw -11585*2
	46	pw_11585_11585x2: times 4 dw 11585*2
	47	int_mshift: db 142, 150, 0, 0, 174, 182, 0, 0
	48	pd_8192: dd 8192
	49	pw_804x2: times 2 dw 804*2
	50	pw_1606x2: times 2 dw 1606*2
	51	pw_3196x2: times 2 dw 3196*2
	52	pw_3981x2: times 2 dw 3981*2
	53	pw_6270x2: times 2 dw 6270*2
	54	pw_7005x2: times 2 dw 7005*2
	55	pw_7723x2: times 2 dw 7723*2
	56	pw_9760x2: times 2 dw 9760*2
	57	pw_12140x2: times 2 dw 12140*2
	58	pw_12665x2: times 2 dw 12665*2
	59	pw_13160x2: times 2 dw 13160*2
	60	pw_13623x2: times 2 dw 13623*2
	61	pw_14053x2: times 2 dw 14053*2
	62	pw_14449x2: times 2 dw 14449*2
	63	pw_14811x2: times 2 dw 14811*2
	64	pw_15137x2: times 2 dw 15137*2
	65	pw_15426x2: times 2 dw 15426*2
	66	pw_15679x2: times 2 dw 15679*2
	67	pw_15893x2: times 2 dw 15893*2
	68	pw_16069x2: times 2 dw 16069*2
	69	pw_16207x2: times 2 dw 16207*2
	70	pw_16305x2: times 2 dw 16305*2
	71	pw_16364x2: times 2 dw 16364*2
	72	pw_m2404x2: times 2 dw -2404*2
	73	pw_m4756x2: times 2 dw -4756*2
	74	pw_m5520x2: times 2 dw -5520*2
	75	pw_m8423x2: times 2 dw -8423*2
	76	pw_m9102x2: times 2 dw -9102*2
	77	pw_m10394x2: times 2 dw -10394*2
	78	pw_m11003x2: times 2 dw -11003*2
	79	pw_804_16364x2: dw 8042, 163642
	80	pw_1606_16305x2: dw 16062, 163052
	81	pw_3196_16069x2: dw 31962, 160692
	82	pw_3981_15893x2: dw 39812, 158932
	83	pw_7005_14811x2: dw 70052, 148112
	84	pw_7723_14449x2: dw 77232, 144492
	85	pw_9760_13160x2: dw 97602, 131602
	86	pw_m2404_16207x2: dw -24042, 162072
	87	pw_m4756_15679x2: dw -47562, 156792
	88	pw_m5520_15426x2: dw -55202, 154262
	89	pw_m8423_14053x2: dw -84232, 140532
	90	pw_m9102_13623x2: dw -91022, 136232
	91	pw_m10394_12665x2: dw -103942, 126652
	92	pw_m11003_12140x2: dw -110032, 121402
	93
	94	%macro COEF_PAIR 2-3 0
	95	%if %3 & 4
	96	pw_%1_m%2: dw %1, -%2
	97	%else
	98	pw_%1_%2: dw %1, %2
	99	%if %3 & 2
	100	pw_m%1_%2: dw -%1, %2
	101	%else
	102	pw_m%2_%1: dw -%2, %1
	103	%endif
	104	%endif
	105	%if %3 & 1
	106	pw_m%1_m%2: dw -%1, -%2
	107	%endif
	108	%endmacro
	109
	110	COEF_PAIR 804, 16364
	111	COEF_PAIR 1606, 16305
	112	COEF_PAIR 3196, 16069, 1
	113	COEF_PAIR 3981, 15893
	114	COEF_PAIR 6270, 15137, 1
	115	COEF_PAIR 7005, 14811
	116	COEF_PAIR 7723, 14449
	117	COEF_PAIR 9102, 13623
	118	COEF_PAIR 9760, 13160
	119	COEF_PAIR 11585, 11585, 1
	120	COEF_PAIR 12140, 11003
	121	COEF_PAIR 12665, 10394
	122	COEF_PAIR 13623, 9102, 1
	123	COEF_PAIR 14053, 8423
	124	COEF_PAIR 15137, 6270
	125	COEF_PAIR 15426, 5520
	126	COEF_PAIR 15679, 4756
	127	COEF_PAIR 16069, 3196
	128	COEF_PAIR 16207, 2404
	129
	130	; ADST16-only:
	131	COEF_PAIR 2404, 9760, 2
	132	COEF_PAIR 5520, 7005, 2
	133	COEF_PAIR 8423, 3981, 2
	134	COEF_PAIR 11003, 804, 2
	135	COEF_PAIR 12140, 16364, 5
	136	COEF_PAIR 14053, 15893, 5
	137	COEF_PAIR 15426, 14811, 5
	138	COEF_PAIR 16207, 13160, 5
	139	pw_11585_m11585: dw 11585, -11585
	140	pw_16069_m3196: dw 16069, -3196
	141	pw_9102_m13623: dw 9102, -13623
	142	pw_15137_m6270: dw 15137, -6270
	143	pw_6270_m15137: dw 6270, -15137
	144
	145	%define pw_11585x2 pw_11585_11585x2
	146	%define pw_m11585x2 pw_m11585_11585x2x4
	147
	148	SECTION .text
	149
	150	%define o_base pw_512 + 128
	151	%define o(x) (r6 - (o_base) + (x))
	152	%define m(x) mangle(private_prefix %+ _ %+ x %+ SUFFIX)
	153
	154	; flags: 1 = swap, 2 = interleave (l), 4 = interleave (t), 8 = no_pack,
	155	; 16 = special_mul1, 32 = special_mul2, 64 = dst_in_tmp1
	156	%macro ITX_MUL2X_PACK 6-7 0 ; dst/src, tmp[1-2], rnd, coef[1-2], flags
	157	mova m%2, m%4
	158	%if %7 & 16
	159	vpdpwssd m%2, m%1, [o(pw_%5)] {bcstd}
	160	mova m%3, m%4
	161	%if %7 & 32
	162	vpdpwssd m%3, m%1, [o(pw_%6)] {bcstd}
	163	%else
	164	vpdpwssd m%3, m%1, m%6
	165	%endif
	166	%elif %7 & 32
	167	vpdpwssd m%2, m%1, m%5
	168	mova m%3, m%4
	169	vpdpwssd m%3, m%1, [o(pw_%6)] {bcstd}
	170	%elif %6 < 32
	171	vpdpwssd m%2, m%1, m%5
	172	mova m%3, m%4
	173	vpdpwssd m%3, m%1, m%6
	174	%elif %7 & 1
	175	vpdpwssd m%2, m%1, [o(pw_%5_%6)] {bcstd}
	176	mova m%3, m%4
	177	vpdpwssd m%3, m%1, [o(pw_m%6_%5)] {bcstd}
	178	%else
	179	vpdpwssd m%2, m%1, [o(pw_m%6_%5)] {bcstd}
	180	mova m%3, m%4
	181	vpdpwssd m%3, m%1, [o(pw_%5_%6)] {bcstd}
	182	%endif
	183	%if %7 & 2
	184	psrld m%2, 14
	185	pslld m%3, 2
	186	vpshrdd m%1, m%3, m%2, 16
	187	%elif %7 & 4
	188	; compared to using shifts (as above) this has better throughput,
	189	; but worse latency and requires setting up the opmask/index
	190	; registers, so only use this method for the larger transforms
	191	%if %7 & 64
	192	pslld m%2, 2
	193	vpmultishiftqb m%2{k7}, m13, m%3
	194	%else
	195	pslld m%1, m%2, 2
	196	vpmultishiftqb m%1{k7}, m13, m%3
	197	%endif
	198	%else
	199	psrad m%2, 14
	200	psrad m%3, 14
	201	%if %7 & 8 == 0
	202	packssdw m%1, m%3, m%2
	203	%endif
	204	%endif
	205	%endmacro
	206
	207	; dst1 = (src1 * coef1 - src2 * coef2 + rnd) >> 12
	208	; dst2 = (src1 * coef2 + src2 * coef1 + rnd) >> 12
	209	%macro ITX_MULSUB_2W 7 ; dst/src[1-2], tmp[1-2], rnd, coef[1-2]
	210	punpcklwd m%3, m%2, m%1
	211	punpckhwd m%2, m%1
	212	%if %7 < 32
	213	mova m%1, m%5
	214	vpdpwssd m%1, m%3, m%7
	215	mova m%4, m%5
	216	vpdpwssd m%4, m%2, m%7
	217	%else
	218	mova m%1, m%5
	219	vpdpwssd m%1, m%3, [o(pw_m%7_%6)] {bcstd}
	220	mova m%4, m%5
	221	vpdpwssd m%4, m%2, [o(pw_m%7_%6)] {bcstd}
	222	%endif
	223	psrad m%1, 14
	224	psrad m%4, 14
	225	packssdw m%1, m%4
	226	mova m%4, m%5
	227	%if %7 < 32
	228	vpdpwssd m%4, m%2, m%6
	229	mova m%2, m%5
	230	vpdpwssd m%2, m%3, m%6
	231	%else
	232	vpdpwssd m%4, m%2, [o(pw_%6_%7)] {bcstd}
	233	mova m%2, m%5
	234	vpdpwssd m%2, m%3, [o(pw_%6_%7)] {bcstd}
	235	%endif
	236	psrad m%4, 14
	237	psrad m%2, 14
	238	packssdw m%2, m%4
	239	%endmacro
	240
	241	; flags: 1 = swap, 2 = invert2, 4 = invert1
	242	%macro ADST_MULSUB_4W 10-11 0 ; dst1/src1, src2, dst2, tmp[1-2], rnd, coef[1-4], flags
	243	mova m%3, m%6
	244	%if %11 & 1
	245	vpdpwssd m%3, m%1, [o(pw_m%8_%7)] {bcstd}
	246	%else
	247	vpdpwssd m%3, m%1, [o(pw_%7_%8)] {bcstd}
	248	%endif
	249	%if %11 & 4
	250	vpbroadcastd m%4, [o(pw_m%9_%10)]
	251	%elif %11 & 2
	252	vpbroadcastd m%4, [o(pw_%9_m%10)]
	253	%elif %11 & 1
	254	vpbroadcastd m%4, [o(pw_%10_%9)]
	255	%else
	256	vpbroadcastd m%4, [o(pw_%9_%10)]
	257	%endif
	258	pmaddwd m%4, m%2
	259	mova m%5, m%6
	260	%if %11 & 4
	261	vpdpwssd m%5, m%1, [o(pw_%8_m%7)] {bcstd}
	262	%elif %11 & 1
	263	vpdpwssd m%5, m%1, [o(pw_%7_%8)] {bcstd}
	264	%else
	265	vpdpwssd m%5, m%1, [o(pw_m%8_%7)] {bcstd}
	266	%endif
	267	%if %11 & 2
	268	vpbroadcastd m%1, [o(pw_%10_%9)]
	269	%elif %11 & 1
	270	vpbroadcastd m%1, [o(pw_%9_m%10)]
	271	%else
	272	vpbroadcastd m%1, [o(pw_m%10_%9)]
	273	%endif
	274	pmaddwd m%2, m%1
	275	paddd m%1, m%3, m%4
	276	psubd m%3, m%4
	277	paddd m%4, m%5, m%2
	278	psubd m%5, m%2
	279	pslld m%1, 2
	280	pslld m%3, 2
	281	vpmultishiftqb m%1{k7}, m13, m%4
	282	vpmultishiftqb m%3{k7}, m13, m%5
	283	%endmacro
	284
	285	%macro WRAP_YMM 1+
	286	INIT_YMM cpuname
	287	%1
	288	INIT_ZMM cpuname
	289	%endmacro
	290
	291	%macro INV_TXFM_FN 3-4 0 ; type1, type2, size, eob_offset
	292	cglobal vp9_i%1_i%2_%3_add, 4, 5, 0, dst, stride, c, eob, tx2
	293	%undef cmp
	294	%define %%p1 m(vp9_i%1_%3_internal)
	295	lea r6, [o_base]
	296	; Jump to the 1st txfm function if we're not taking the fast path, which
	297	; in turn performs an indirect jump to the 2nd txfm function.
	298	lea tx2q, [m(vp9_i%2_%3_internal).pass2]
	299	%ifidn %1_%2, dct_dct
	300	cmp eobd, 1
	301	jne %%p1
	302	%else
	303	%if %4
	304	add eobd, %4
	305	%endif
	306	; jump to the 1st txfm function unless it's located directly after this
	307	times ((%%end - %%p1) >> 31) & 1 jmp %%p1
	308	ALIGN function_align
	309	%%end:
	310	%endif
	311	%endmacro
	312
	313	%macro INV_TXFM_16X16_FN 2-3 0 ; type1, type2, eob_offset
	314	INV_TXFM_FN %1, %2, 16x16, %3
	315	%ifidn %1_%2, dct_dct
	316	movd xmm0, [o(pw_11585x2)]
	317	pmulhrsw xmm3, xmm0, [cq]
	318	pxor ym2, ym2
	319	pmulhrsw xmm3, xmm0
	320	pmulhrsw xmm3, [o(pw_512)]
	321	mova [cq], xm2
	322	add r3d, 7
	323	vpbroadcastw ym3, xmm3
	324	.dconly_loop:
	325	mova xm1, [dstq+strideq*0]
	326	vinserti32x4 ym1, [dstq+strideq*1], 1
	327	punpcklbw ym0, ym1, ym2
	328	punpckhbw ym1, ym2
	329	paddw ym0, ym3
	330	paddw ym1, ym3
	331	packuswb ym0, ym1
	332	mova [dstq+strideq*0], xm0
	333	vextracti32x4 [dstq+strideq*1], ym0, 1
	334	lea dstq, [dstq+strideq*2]
	335	dec r3d
	336	jg .dconly_loop
	337	RET
	338	%endif
	339	%endmacro
	340
	341	%macro IDCT16_MAIN 0-1 0 ; idct32
	342	%if mmsize == 64 && %1 == 0
	343	.main_fast:
	344	%endif
	345	vpbroadcastd m2, [o(pw_1606_16305x2)]
	346	vpbroadcastd m4, [o(pw_m10394_12665x2)]
	347	vpbroadcastd m11, [o(pw_7723_14449x2)]
	348	vpbroadcastd m12, [o(pw_m4756_15679x2)]
	349	pmulhrsw m8, m2 ; t8a t15a
	350	vpbroadcastd m2, [o(pw_3196_16069x2)]
	351	pmulhrsw m0, m4 ; t9a t14a
	352	vpbroadcastd m4, [o(pw_m9102_13623x2)]
	353	pmulhrsw m5, m11 ; t10a t13a
	354	vpbroadcastd m11, [o(pw_11585_11585x2)]
	355	pmulhrsw m1, m12 ; t11a t12a
	356	vbroadcasti32x4 m12, [o(pw_15137_6270x2x4)]
	357	pmulhrsw m7, m2 ; t4a t7a
	358	pmulhrsw m3, m4 ; t5a t6a
	359	pmulhrsw m9, m11 ; t0 t1
	360	pmulhrsw m6, m12 ; t3 t2
	361	%if mmsize == 64 && %1 == 0
	362	jmp %%main2
	363	ALIGN function_align
	364	.main:
	365	punpckhwd m8, m7, m0 ; dct16 in15 in1
	366	punpcklwd m9, m4, m0 ; dct4 in2 in0
	367	punpckhwd m0, m3, m4 ; dct16 in7 in9
	368	punpcklwd m7, m1 ; dct8 in7 in1
	369	punpckhwd m1, m6 ; dct16 in3 in13
	370	punpcklwd m3, m5 ; dct8 in3 in5
	371	punpckhwd m5, m2 ; dct16 in11 in5
	372	punpcklwd m6, m2 ; dct4 in3 in1
	373	ITX_MUL2X_PACK 8, 2, 4, 10, 1606, 16305, 5 ; t8a t15a
	374	ITX_MUL2X_PACK 0, 2, 4, 10, 12665, 10394, 5 ; t9a t14a
	375	ITX_MUL2X_PACK 5, 2, 4, 10, 7723, 14449, 5 ; t10a t13a
	376	ITX_MUL2X_PACK 1, 2, 4, 10, 15679, 4756, 5 ; t11a t12a
	377	ITX_MUL2X_PACK 7, 2, 4, 10, 3196, 16069, 5 ; t4a t7a
	378	ITX_MUL2X_PACK 3, 2, 4, 10, 13623, 9102, 5 ; t5a t6a
	379	ITX_MUL2X_PACK 9, 2, 4, 10, 11585, 11585 ; t0 t1
	380	ITX_MUL2X_PACK 6, 2, 4, 10, 6270, 15137 ; t3 t2
	381	%%main2:
	382	%endif
	383	psubw m2, m8, m0 ; t9 t14
	384	paddw m8, m0 ; t8 t15
	385	psubw m4, m1, m5 ; t10 t13
	386	paddw m1, m5 ; t11 t12
	387	ITX_MUL2X_PACK 2, 0, 5, 10, 6270, 15137, (1\|%1*4) ; t9a t14a
	388	ITX_MUL2X_PACK 4, 0, 5, 10, m15137, 6270, (1\|%1*4) ; t10a t13a
	389	vbroadcasti32x4 m5, [o(deint_shuf)]
	390	psubw m0, m8, m1 ; t11a t12a
	391	paddw m8, m1 ; t8a t15a
	392	psubw m1, m7, m3 ; t5a t6a
	393	paddw m7, m3 ; t4 t7
	394	pshufb m8, m5
	395	pshufb m7, m5
	396	paddw m3, m2, m4 ; t9 t14
	397	psubw m2, m4 ; t10 t13
	398	%if %1
	399	vpbroadcastd m12, [o(pw_11585_11585)]
	400	vpbroadcastd m11, [o(pw_m11585_11585)]
	401	pshufb m3, m5
	402	ITX_MUL2X_PACK 1, 4, 5, 10, 12, 11 ; t5 t6
	403	ITX_MUL2X_PACK 0, 4, 5, 10, 11, 12, 8 ; t11 t12
	404	ITX_MUL2X_PACK 2, 0, 11, 10, 11, 12, 8 ; t10a t13a
	405	packssdw m5, m11 ; t12 t13a
	406	packssdw m4, m0 ; t11 t10a
	407	%else
	408	pshufb m0, m5
	409	ITX_MUL2X_PACK 1, 4, 5, 10, 11585_11585, m11585_11585, 48 ; t5 t6
	410	vpbroadcastd m11, [o(pw_11585x2)]
	411	punpckhqdq m5, m0, m2 ; t12a t13
	412	punpcklqdq m0, m2 ; t11a t10
	413	psubw m4, m5, m0
	414	paddw m5, m0
	415	pmulhrsw m4, m11 ; t11 t10a
	416	pmulhrsw m5, m11 ; t12 t13a
	417	%endif
	418	punpckhqdq m2, m7, m1 ; t7 t6
	419	punpcklqdq m7, m1 ; t4 t5
	420	psubw m1, m9, m6 ; t3 t2
	421	paddw m9, m6 ; t0 t1
	422	punpckhqdq m0, m8, m3 ; t15a t14
	423	punpcklqdq m8, m3 ; t8a t9
	424	psubw m3, m9, m2 ; t7 t6
	425	paddw m9, m2 ; t0 t1
	426	psubw m2, m1, m7 ; t4 t5
	427	paddw m1, m7 ; t3 t2
	428	psubw m7, m9, m0 ; out15 out14
	429	paddw m0, m9 ; out0 out1
	430	psubw m6, m1, m5 ; out12 out13
	431	paddw m1, m5 ; out3 out2
	432	psubw m5, m2, m4 ; out11 out10
	433	paddw m2, m4 ; out4 out5
	434	psubw m4, m3, m8 ; out8 out9
	435	paddw m3, m8 ; out7 out6
	436	%endmacro
	437
	438	INIT_ZMM avx512icl
	439	INV_TXFM_16X16_FN dct, dct
	440	INV_TXFM_16X16_FN dct, adst, 39-23
	441
	442	cglobal vp9_idct_16x16_internal, 0, 5, 16, dst, stride, c, eob, tx2
	443	mova m15, [o(itx_perm)]
	444	vpbroadcastd m10, [o(pd_8192)]
	445	vpbroadcastq m13, [o(int_mshift)]
	446	vpcmpub k7, m13, m10, 6
	447	sub eobd, 39
	448	jl .pass1_fast
	449	vpermq m0, m15, [cq+64*0]
	450	vpermq m1, m15, [cq+64*1]
	451	vpermq m2, m15, [cq+64*2]
	452	vpermq m3, m15, [cq+64*3]
	453	vpermq m4, m15, [cq+64*4]
	454	vpermq m5, m15, [cq+64*5]
	455	vpermq m6, m15, [cq+64*6]
	456	vpermq m7, m15, [cq+64*7]
	457	call .main
	458	vbroadcasti32x4 m12, [o(int_shuf1)]
	459	vbroadcasti32x4 m11, [o(int_shuf2)]
	460	pshufb m0, m12
	461	pshufb m8, m1, m11
	462	pshufb m2, m12
	463	pshufb m9, m3, m11
	464	pshufb m4, m12
	465	pshufb m14, m5, m11
	466	pshufb m6, m12
	467	pshufb m11, m7, m11
	468	punpckhdq m1, m0, m8
	469	punpckldq m0, m8
	470	punpckhdq m3, m2, m9
	471	punpckldq m2, m9
	472	punpckhdq m5, m4, m14
	473	punpckldq m4, m14
	474	punpckhdq m7, m6, m11
	475	punpckldq m6, m11
	476	.pass1_end:
	477	vshufi32x4 m8, m4, m6, q3232
	478	vinserti32x8 m4, ym6, 1
	479	vshufi32x4 m6, m0, m2, q3232
	480	vinserti32x8 m0, ym2, 1
	481	vshufi32x4 m9, m5, m7, q3232
	482	vinserti32x8 m5, ym7, 1
	483	vshufi32x4 m7, m1, m3, q3232
	484	vinserti32x8 m1, ym3, 1
	485	vshufi32x4 m2, m0, m4, q3131 ; 4 5
	486	vshufi32x4 m0, m4, q2020 ; 0 1
	487	vshufi32x4 m4, m6, m8, q2020 ; 8 9
	488	vshufi32x4 m6, m8, q3131 ; 12 13
	489	vshufi32x4 m3, m1, m5, q3131 ; 6 7
	490	vshufi32x4 m1, m5, q2020 ; 2 3
	491	vshufi32x4 m5, m7, m9, q2020 ; 10 11
	492	vshufi32x4 m7, m9, q3131 ; 14 1
	493	jmp tx2q
	494	.pass1_fast:
	495	mova ym3, [o(dup16_perm)]
	496	vbroadcasti32x4 ym9, [cq+32*0]
	497	vbroadcasti32x4 ym6, [cq+32*4]
	498	vpermb ym8, ym3, [cq+32*1]
	499	vpermb ym0, ym3, [cq+32*7]
	500	vpermb ym5, ym3, [cq+32*5]
	501	vpermb ym1, ym3, [cq+32*3]
	502	vpermb ym7, ym3, [cq+32*2]
	503	vpermb ym3, ym3, [cq+32*6]
	504	shufpd ym9, ym9, 0x0c
	505	shufpd ym6, ym6, 0x0c
	506	WRAP_YMM IDCT16_MAIN
	507	vbroadcasti32x4 m8, [o(int_shuf1)]
	508	vbroadcasti32x4 m9, [o(int_shuf2)]
	509	vinserti32x8 m0, ym2, 1 ; 0 1 \| 4 5
	510	vinserti32x8 m4, ym6, 1 ; 8 9 \| 12 13
	511	vinserti32x8 m1, ym3, 1 ; 3 2 \| 7 6
	512	vinserti32x8 m5, ym7, 1 ; 11 10 \| 15 14
	513	vshufi32x4 m2, m0, m4, q3131
	514	vshufi32x4 m0, m4, q2020
	515	vshufi32x4 m4, m1, m5, q2020
	516	vshufi32x4 m1, m5, q3131
	517	pshufb m2, m8
	518	pshufb m0, m8
	519	pshufb m4, m9
	520	pshufb m1, m9
	521	punpckhdq m3, m2, m1 ; 6-7
	522	punpckldq m2, m1 ; 4-5
	523	punpckhdq m1, m0, m4 ; 2-3
	524	punpckldq m0, m4 ; 0-1
	525	jmp tx2q
	526	.pass2:
	527	test eobd, eobd
	528	jl .pass2_fast
	529	call .main
	530	jmp .pass2_end
	531	.pass2_fast:
	532	punpcklqdq m9, m0, m0
	533	punpckhwd m8, m0, m0
	534	punpcklwd m7, m1, m1
	535	punpckhwd m1, m1
	536	punpcklqdq m6, m2, m2
	537	punpckhwd m5, m2, m2
	538	punpckhwd m0, m3, m3
	539	punpcklwd m3, m3
	540	call .main_fast
	541	.pass2_end:
	542	psrldq m8, m15, 1
	543	psrlq m12, m15, 12
	544	psrldq m9, m15, 2
	545	psrlq m13, m15, 20
	546	mova m10, m8
	547	vpermi2q m8, m0, m2 ; 0 1 4 5
	548	vpermt2q m0, m12, m2
	549	mova m11, m9
	550	vpermi2q m9, m1, m3 ; 2 3 6 7
	551	vpermt2q m1, m13, m3
	552	vpbroadcastd m2, [o(pw_512)]
	553	vpermi2q m10, m4, m6 ; 8 9 12 13
	554	vpermt2q m4, m12, m6
	555	vpermi2q m11, m5, m7 ; 10 11 14 15
	556	vpermt2q m5, m13, m7
	557	REPX {pmulhrsw x, m2}, m0, m1, m4, m5, m8, m9, m10, m11
	558	.pass2_end2:
	559	lea r3, [strideq*3]
	560	lea r4, [dstq+strideq*4]
	561	lea r5, [dstq+strideq*8]
	562	lea r6, [r4 +strideq*8]
	563	mova xm3, [dstq+strideq*0]
	564	mova xm6, [dstq+strideq*2]
	565	vinserti32x4 ym3, [dstq+strideq*1], 1
	566	vinserti32x4 ym6, [dstq+r3 ], 1
	567	vinserti32x4 m3, [r4+strideq*0], 2
	568	vinserti32x4 m6, [r4+strideq*2], 2
	569	vinserti32x4 m3, [r4+strideq*1], 3
	570	vinserti32x4 m6, [r4+r3 ], 3
	571	mova xm12, [r5+strideq*0]
	572	mova xm13, [r5+strideq*2]
	573	vinserti32x4 ym12, [r5+strideq*1], 1
	574	vinserti32x4 ym13, [r5+r3 ], 1
	575	vinserti32x4 m12, [r6+strideq*0], 2
	576	vinserti32x4 m13, [r6+strideq*2], 2
	577	vinserti32x4 m12, [r6+strideq*1], 3
	578	vinserti32x4 m13, [r6+r3 ], 3
	579	pxor m7, m7
	580	REPX {mova [cq+64*x], m7}, 0, 1, 2, 3, 4, 5, 6, 7
	581	punpcklbw m2, m3, m7
	582	punpckhbw m3, m7
	583	paddw m0, m2
	584	paddw m8, m3
	585	packuswb m0, m8
	586	punpcklbw m2, m6, m7
	587	punpckhbw m6, m7
	588	paddw m1, m2
	589	paddw m9, m6
	590	packuswb m1, m9
	591	punpcklbw m2, m12, m7
	592	punpckhbw m12, m7
	593	paddw m2, m4
	594	paddw m10, m12
	595	packuswb m2, m10
	596	punpcklbw m3, m13, m7
	597	punpckhbw m13, m7
	598	paddw m3, m5
	599	paddw m11, m13
	600	packuswb m3, m11
	601	mova [dstq+strideq*0], xm0
	602	vextracti32x4 [dstq+strideq*1], ym0, 1
	603	mova [dstq+strideq*2], xm1
	604	vextracti32x4 [dstq+r3 ], ym1, 1
	605	vextracti32x4 [r4+strideq*0], m0, 2
	606	vextracti32x4 [r4+strideq*1], m0, 3
	607	vextracti32x4 [r4+strideq*2], m1, 2
	608	vextracti32x4 [r4+r3 ], m1, 3
	609	mova [r5+strideq*0], xm2
	610	vextracti32x4 [r5+strideq*1], ym2, 1
	611	mova [r5+strideq*2], xm3
	612	vextracti32x4 [r5+r3 ], ym3, 1
	613	vextracti32x4 [r6+strideq*0], m2, 2
	614	vextracti32x4 [r6+strideq*1], m2, 3
	615	vextracti32x4 [r6+strideq*2], m3, 2
	616	vextracti32x4 [r6+r3 ], m3, 3
	617	RET
	618	ALIGN function_align
	619	IDCT16_MAIN
	620	ret
	621
	622	%macro IADST16_MAIN 0
	623	%if mmsize == 64
	624	.main_fast:
	625	%endif
	626	punpcklwd m4, m3, m0 ; in7 in0
	627	punpcklwd m11, m1, m2 ; in3 in4
	628	punpckhwd m9, m2, m1 ; in5 in2
	629	punpckhwd m7, m0, m3 ; in1 in6
	630	ITX_MUL2X_PACK 4, 0, 6, 10, 11003_804, 12140_m16364, 116 ; t1a t0a
	631	ITX_MUL2X_PACK 4, 5, 6, 10, m11003_804, m12140_m16364, 52 ; t9a t8a
	632	ITX_MUL2X_PACK 11, 2, 6, 10, 5520_7005, 15426_m14811, 116 ; t5a t4a
	633	ITX_MUL2X_PACK 11, 5, 6, 10, m5520_7005, m15426_m14811, 52 ; t13a t12a
	634	ITX_MUL2X_PACK 9, 1, 6, 10, 8423_3981, 14053_m15893, 116 ; t3a t2a
	635	ITX_MUL2X_PACK 9, 5, 6, 10, m8423_3981, m14053_m15893, 52 ; t11a t10a
	636	ITX_MUL2X_PACK 7, 3, 6, 10, 2404_9760, 16207_m13160, 116 ; t7a t6a
	637	ITX_MUL2X_PACK 7, 5, 6, 10, m2404_9760, m16207_m13160, 52 ; t15a t14a
	638	%if mmsize == 64 ; for the ymm variant we only ever use the fast path
	639	jmp %%main2
	640	ALIGN function_align
	641	.main:
	642	punpckhwd m8, m7, m0 ; in14 in1
	643	punpcklwd m0, m7 ; in0 in15
	644	punpcklwd m7, m6, m1 ; in12 in3
	645	punpckhwd m1, m6 ; in2 in13
	646	punpckhwd m6, m5, m2 ; in10 in5
	647	punpcklwd m2, m5 ; in4 in11
	648	punpcklwd m5, m4, m3 ; in8 in7
	649	punpckhwd m3, m4 ; in6 in9
	650	ADST_MULSUB_4W 0, 5, 4, 9, 11, 10, 804, 16364, 12140, 11003 ; t1a t0a, t9a t8a
	651	ADST_MULSUB_4W 2, 7, 11, 5, 9, 10, 7005, 14811, 15426, 5520 ; t5a t4a, t13a t12a
	652	ADST_MULSUB_4W 1, 6, 9, 5, 7, 10, 3981, 15893, 14053, 8423 ; t3a t2a, t11a t10a
	653	ADST_MULSUB_4W 3, 8, 7, 5, 6, 10, 9760, 13160, 16207, 2404 ; t7a t6a, t15a t14a
	654	%%main2:
	655	%endif
	656	psubw m5, m1, m3 ; t7 t6
	657	paddw m6, m1, m3 ; t3 t2
	658	psubw m1, m0, m2 ; t5 t4
	659	paddw m2, m0 ; t1 t0
	660	ADST_MULSUB_4W 4, 11, 8, 3, 0, 10, 3196, 16069, 16069, 3196, 1 ; t8a t9a, t12a t13a
	661	ADST_MULSUB_4W 9, 7, 0, 3, 11, 10, 13623, 9102, 9102, 13623, 1 ; t10a t11a, t14a t15a
	662	ADST_MULSUB_4W 1, 5, 11, 3, 7, 10, 6270, 15137, 15137, 6270, 2 ; out12 -out3, t7 t6
	663	psubw m3, m2, m6 ; t3a t2a
	664	paddw m2, m6 ; -out15 out0
	665	ADST_MULSUB_4W 8, 0, 5, 6, 7, 10, 15137, 6270, 6270, 15137, 6 ; -out13 out2, t15a t14
	666	vbroadcasti32x4 m12, [o(deint_shuf)]
	667	paddw m0, m4, m9 ; -out1 out14
	668	psubw m4, m9 ; t10 t11
	669	pshufb m2, m12
	670	pshufb m1, m12
	671	pshufb m8, m12
	672	pshufb m0, m12
	673	punpcklqdq m6, m1, m8 ; out12 -out13
	674	shufps m7, m0, m2, q1032 ; out14 -out15
	675	%endmacro
	676
	677	%macro IADST16_PASS1_END 0
	678	shufps m0, m2, m0, q1032 ; out0 -out1
	679	punpckhqdq m1, m8, m1 ; out2 -out3
	680	mova m2, m10
	681	vpdpwssd m2, m5, [o(pw_m11585_m11585)] {bcstd} ; out5
	682	mova m8, m10
	683	vpdpwssd m8, m11, [o(pw_11585_11585)] {bcstd} ; out4
	684	mova m9, m10
	685	vpdpwssd m9, m5, [o(pw_m11585_11585)] {bcstd} ; out10
	686	mova m5, m10
	687	vpdpwssd m5, m11, [o(pw_11585_m11585)] {bcstd} ; out11
	688	mova m11, m10
	689	vpdpwssd m11, m3, [o(pw_m11585_m11585)] {bcstd} ; out7
	690	mova m14, m10
	691	vpdpwssd m14, m4, [o(pw_11585_11585)] {bcstd} ; out6
	692	mova m12, m10
	693	vpdpwssd m12, m3, [o(pw_m11585_11585)] {bcstd} ; out8
	694	mova m3, m10
	695	vpdpwssd m3, m4, [o(pw_m11585_11585)] {bcstd} ; out9
	696	%endmacro
	697
	698	INV_TXFM_16X16_FN adst, dct, 39-18
	699	INV_TXFM_16X16_FN adst, adst
	700
	701	cglobal vp9_iadst_16x16_internal, 0, 5, 16, dst, stride, c, eob, tx2
	702	mova m15, [o(itx_perm)]
	703	psrlq m7, m15, 4
	704	vpermq m0, m15, [cq+64*0] ; 0 1
	705	vpermq m1, m7, [cq+64*1] ; 3 2
	706	vpermq m2, m15, [cq+64*2] ; 4 5
	707	vpermq m3, m7, [cq+64*3] ; 7 6
	708	vpbroadcastd m10, [o(pd_8192)]
	709	vpbroadcastq m13, [o(int_mshift)]
	710	vpcmpub k7, m13, m10, 6
	711	sub eobd, 39
	712	jl .pass1_fast
	713	vpermq m4, m15, [cq+64*4] ; 8 9
	714	vpermq m5, m7, [cq+64*5] ; 11 10
	715	vpermq m6, m15, [cq+64*6] ; 12 13
	716	vpermq m7, m7, [cq+64*7] ; 15 14
	717	call .main
	718	IADST16_PASS1_END
	719	REPX {psrad x, 14}, m2, m8, m9, m5, m11, m14, m12, m3
	720	packssdw m2, m8, m2 ; out4 out5
	721	packssdw m5, m9, m5 ; out10 out11
	722	packssdw m4, m12, m3 ; out8 out9
	723	packssdw m3, m14, m11 ; out6 out7
	724	pxor m9, m9
	725	punpckhwd m8, m0, m1
	726	punpcklwd m0, m1
	727	psubw m8, m9, m8
	728	punpckhwd m1, m0, m8
	729	punpcklwd m0, m8
	730	punpckhwd m8, m2, m3
	731	punpcklwd m2, m3
	732	punpckhwd m3, m2, m8
	733	punpcklwd m2, m8
	734	punpckhwd m8, m4, m5
	735	punpcklwd m4, m5
	736	punpckhwd m5, m4, m8
	737	punpcklwd m4, m8
	738	punpckhwd m8, m6, m7
	739	punpcklwd m6, m7
	740	psubw m8, m9, m8
	741	punpckhwd m7, m6, m8
	742	punpcklwd m6, m8
	743	jmp m(vp9_idct_16x16_internal).pass1_end
	744	.pass1_fast:
	745	WRAP_YMM IADST16_MAIN
	746	WRAP_YMM IADST16_PASS1_END
	747	vinserti32x8 m0, ym6, 1
	748	vinserti32x8 m1, ym7, 1
	749	vinserti32x8 m8, ym12, 1
	750	vinserti32x8 m2, ym3, 1
	751	vinserti32x8 m14, ym9, 1
	752	vinserti32x8 m11, ym5, 1
	753	pslld m14, 2
	754	pslld m11, 2
	755	punpckhwd m4, m0, m1
	756	punpcklwd m0, m1
	757	vpmultishiftqb m14{k7}, m13, m8
	758	vpmultishiftqb m11{k7}, m13, m2
	759	psrlq m1, m15, 24
	760	pxor m2, m2
	761	psubw m2, m4
	762	punpckhwd m3, m0, m2
	763	punpcklwd m0, m2
	764	psrlq m2, m15, 28
	765	punpckhwd m4, m14, m11
	766	punpcklwd m14, m11
	767	mova m5, m2
	768	vpermi2q m2, m0, m14
	769	vpermt2q m0, m1, m14
	770	vpermi2q m1, m3, m4
	771	vpermt2q m3, m5, m4
	772	jmp tx2q
	773	.pass2:
	774	pshufd m1, m1, q1032
	775	pshufd m3, m3, q1032
	776	test eobd, eobd
	777	jl .pass2_fast
	778	pshufd m5, m5, q1032
	779	pshufd m7, m7, q1032
	780	call .main
	781	jmp .pass2_end
	782	.pass2_fast:
	783	call .main_fast
	784	.pass2_end:
	785	vbroadcasti32x4 m9, [o(pw_11585_m11585x2x4)]
	786	vbroadcasti32x4 m10, [o(pw_m11585_11585x2x4)]
	787	punpckhqdq m1, m8 ; -out3 out2
	788	shufps m0, m2, q3210 ; -out1 out0
	789	pshufb m2, m11, m12
	790	pshufb m5, m12
	791	pshufb m3, m12
	792	pshufb m4, m12
	793	vbroadcasti32x4 m11, [o(pw_512)]
	794	vpbroadcastd m12, [o(pw_512)]
	795	punpcklqdq m8, m5, m2 ; t15a t7
	796	punpckhqdq m5, m2 ; t14a t6
	797	shufps m2, m3, m4, q1032 ; t2a t10
	798	shufps m3, m4, q3210 ; t3a t11
	799	psubsw m4, m2, m3
	800	paddsw m3, m2
	801	paddsw m2, m5, m8
	802	psubsw m5, m8
	803	pmulhrsw m4, m9 ; out8 out9
	804	pmulhrsw m3, m10 ; out7 out6
	805	pmulhrsw m2, m10 ; out5 out4
	806	pmulhrsw m5, m9 ; out10 out11
	807	pmulhrsw m6, m11
	808	pmulhrsw m7, m11
	809	pshufd m11, m11, q1032
	810	pmulhrsw m0, m11
	811	pmulhrsw m1, m11
	812	REPX {pmulhrsw x, m12}, m2, m3, m4, m5
	813	psrldq m8, m15, 2
	814	psrlq m12, m15, 20
	815	psrldq m10, m15, 1
	816	psrlq m13, m15, 12
	817	mova m9, m8
	818	vpermi2q m8, m0, m2 ; 0 1 4 5
	819	vpermt2q m0, m12, m2
	820	vpermi2q m9, m1, m3 ; 2 3 6 7
	821	vpermt2q m1, m12, m3
	822	mova m11, m10
	823	vpermi2q m10, m4, m6 ; 8 9 12 13
	824	vpermt2q m4, m13, m6
	825	vpermi2q m11, m5, m7 ; 10 11 14 15
	826	vpermt2q m5, m13, m7
	827	jmp m(vp9_idct_16x16_internal).pass2_end2
	828	ALIGN function_align
	829	IADST16_MAIN
	830	ret
	831
	832	%macro IDCT_32x32_END 4 ; src, mem, stride[1-2]
	833	pmovzxbw m10, [dstq+%3]
	834	pmovzxbw m11, [r3 +%4]
	835	%if %2 < 8
	836	paddw m8, m%2, m%1
	837	psubw m9, m%2, m%1
	838	%else
	839	mova m9, [rsp+64*(%2-8)]
	840	paddw m8, m9, m%1
	841	psubw m9, m%1
	842	%endif
	843	pmulhrsw m8, m12
	844	pmulhrsw m9, m12
	845	paddw m8, m10
	846	paddw m9, m11
	847	packuswb m8, m9
	848	vpermq m8, m13, m8
	849	mova [dstq+%3], ym8
	850	vextracti32x8 [r3 +%4], m8, 1
	851	%if %2 == 3 \|\| %2 == 7 \|\| %2 == 11
	852	add dstq, r5
	853	sub r3, r5
	854	%endif
	855	%endmacro
	856
	857	cglobal vp9_idct_idct_32x32_add, 4, 7, 0, dst, stride, c, eob
	858	%undef cmp
	859	lea r6, [o_base]
	860	cmp eobd, 1
	861	jne .pass1
	862	movd xmm0, [o(pw_11585x2)]
	863	pmulhrsw xmm3, xmm0, [cq]
	864	pxor m2, m2
	865	pmulhrsw xmm3, xmm0
	866	pmulhrsw xmm3, [o(pw_512)]
	867	movd [cq], xm2
	868	add r3d, 15
	869	vpbroadcastw m3, xmm3
	870	.dconly_loop:
	871	mova ym1, [dstq+strideq*0]
	872	vinserti32x8 m1, [dstq+strideq*1], 1
	873	punpcklbw m0, m1, m2
	874	punpckhbw m1, m2
	875	paddw m0, m3
	876	paddw m1, m3
	877	packuswb m0, m1
	878	mova [dstq+strideq*0], ym0
	879	vextracti32x8 [dstq+strideq*1], m0, 1
	880	lea dstq, [dstq+strideq*2]
	881	dec r3d
	882	jg .dconly_loop
	883	RET
	884	.pass1:
	885	PROLOGUE 0, 7, 30, 64*16, dst, stride, c, eob
	886	sub eobd, 135
	887	jl .fast
	888	mova m0, [cq+64* 0]
	889	mova m14, [cq+64* 2]
	890	mova m1, [cq+64* 4]
	891	mova m15, [cq+64* 6]
	892	mova m2, [cq+64* 8]
	893	mova m16, [cq+64*10]
	894	mova m3, [cq+64*12]
	895	mova m17, [cq+64*14]
	896	mova m4, [cq+64*16]
	897	mova m18, [cq+64*18]
	898	mova m5, [cq+64*20]
	899	mova m19, [cq+64*22]
	900	mova m6, [cq+64*24]
	901	mova m20, [cq+64*26]
	902	mova m7, [cq+64*28]
	903	mova m21, [cq+64*30]
	904	call .idct16
	905	mova [rsp+64*0], m14
	906	mova [rsp+64*1], m15
	907	mova [rsp+64*2], m16
	908	mova [rsp+64*3], m17
	909	mova [rsp+64*4], m18
	910	mova [rsp+64*5], m19
	911	mova [rsp+64*6], m20
	912	mova [rsp+64*7], m21
	913	mova m22, [cq+64* 1]
	914	mova m23, [cq+64* 3]
	915	mova m24, [cq+64* 5]
	916	mova m25, [cq+64* 7]
	917	mova m26, [cq+64* 9]
	918	mova m27, [cq+64*11]
	919	mova m28, [cq+64*13]
	920	mova m29, [cq+64*15]
	921	mova m14, [cq+64*17]
	922	mova m15, [cq+64*19]
	923	mova m16, [cq+64*21]
	924	mova m17, [cq+64*23]
	925	mova m18, [cq+64*25]
	926	mova m19, [cq+64*27]
	927	mova m20, [cq+64*29]
	928	mova m21, [cq+64*31]
	929	call .main
	930	psubw m13, m0, m29 ; 31
	931	paddw m0, m29 ; 0
	932	psubw m29, m1, m28 ; 30
	933	paddw m1, m28 ; 1
	934	psubw m28, m2, m27 ; 29
	935	paddw m2, m27 ; 2
	936	psubw m27, m3, m26 ; 28
	937	paddw m3, m26 ; 3
	938	psubw m26, m4, m25 ; 27
	939	paddw m4, m25 ; 4
	940	psubw m25, m5, m24 ; 26
	941	paddw m5, m24 ; 5
	942	psubw m24, m6, m23 ; 25
	943	paddw m6, m23 ; 6
	944	psubw m23, m7, m22 ; 24
	945	paddw m7, m22 ; 7
	946	punpckhwd m8, m0, m1 ; a4 b4 a5 b5 a6 b6 a7 b7
	947	punpcklwd m0, m1 ; a0 b0 a1 b1 a2 b2 a3 b3
	948	punpckhwd m1, m2, m3 ; c4 d4 c5 d5 c6 d6 c7 d7
	949	punpcklwd m2, m3 ; c0 d0 c1 d1 c2 d2 c3 d3
	950	punpckhwd m22, m4, m5 ; e4 f4 e5 f5 e6 f6 e7 f7
	951	punpcklwd m4, m5 ; e0 f0 e1 f1 e2 f2 e3 f3
	952	punpckhwd m5, m6, m7 ; g4 h4 g5 h5 g6 h6 g7 h7
	953	punpcklwd m6, m7 ; g0 h0 g1 h1 g2 h2 g3 h3
	954	punpckhwd m3, m23, m24
	955	punpcklwd m23, m24
	956	punpckhwd m24, m25, m26
	957	punpcklwd m25, m26
	958	punpckhwd m26, m27, m28
	959	punpcklwd m27, m28
	960	punpckhwd m28, m29, m13
	961	punpcklwd m29, m13
	962	punpckhdq m7, m0, m2 ; a2 b2 c2 d2 a3 b3 c3 d3
	963	punpckldq m0, m2 ; a0 b0 c0 d0 a1 b1 c1 d1
	964	punpckhdq m2, m4, m6 ; e2 f2 g2 h2 e3 f3 g3 h3
	965	punpckldq m4, m6 ; e0 f0 g0 h0 e1 f1 g1 h1
	966	punpckhdq m6, m8, m1 ; a6 b6 c6 d6 a7 b7 c7 d7
	967	punpckldq m8, m1 ; a4 b4 c4 d4 a5 b5 c5 d5
	968	punpckhdq m1, m22, m5 ; e6 f6 g6 h6 e7 f7 g7 h7
	969	punpckldq m22, m5 ; e4 f4 g4 h5 e5 f5 g5 h5
	970	punpckhdq m13, m23, m25
	971	punpckldq m23, m25
	972	punpckhdq m25, m27, m29
	973	punpckldq m27, m29
	974	punpckhdq m9, m3, m24
	975	punpckldq m3, m24
	976	punpckhdq m24, m26, m28
	977	punpckldq m26, m28
	978	punpcklqdq m5, m23, m27 ; d00 d08 d16 d24
	979	punpckhqdq m23, m27 ; d01 d09 d17 d25
	980	punpckhqdq m27, m13, m25 ; d03 d11 d19 d27
	981	punpcklqdq m13, m25 ; d02 d10 d18 d26
	982	punpckhqdq m25, m3, m26 ; d05 d13 d21 d29
	983	punpcklqdq m3, m26 ; d04 d12 d20 d28
	984	punpckhqdq m26, m9, m24 ; d07 d15 d23 d31
	985	punpcklqdq m9, m24 ; d06 d14 d22 d30
	986	mova [rsp+64*12], m23
	987	mova [rsp+64*13], m27
	988	mova [rsp+64*14], m25
	989	mova [rsp+64*15], m26
	990	punpckhqdq m24, m8, m22 ; a05 a13 a21 a29
	991	punpcklqdq m8, m22 ; a04 a12 a20 a28
	992	punpckhqdq m22, m0, m4 ; a01 a09 a17 a25
	993	punpcklqdq m0, m4 ; a00 a08 a16 a24
	994	punpckhqdq m23, m7, m2 ; a03 a11 a19 a27
	995	punpcklqdq m7, m2 ; a02 a10 a18 a26
	996	punpckhqdq m25, m6, m1 ; a07 a15 a23 a31
	997	punpcklqdq m6, m1 ; a06 a14 a22 a30
	998	mova m2, [rsp+64*0]
	999	mova m11, [rsp+64*1]
	1000	mova m12, [rsp+64*2]
	1001	mova m29, [rsp+64*3]
	1002	mova m27, [rsp+64*4]
	1003	mova m26, [rsp+64*5]
	1004	mova m4, [rsp+64*6]
	1005	mova m28, [rsp+64*7]
	1006	psubw m1, m2, m21 ; 23
	1007	paddw m2, m21 ; 8
	1008	psubw m21, m11, m20 ; 22
	1009	paddw m11, m20 ; 9
	1010	psubw m20, m12, m19 ; 21
	1011	paddw m12, m19 ; 10
	1012	psubw m19, m29, m18 ; 20
	1013	paddw m29, m18 ; 11
	1014	psubw m18, m27, m17 ; 19
	1015	paddw m27, m17 ; 12
	1016	psubw m17, m26, m16 ; 18
	1017	paddw m26, m16 ; 13
	1018	paddw m16, m4, m15 ; 14
	1019	psubw m4, m15 ; 17
	1020	mova m15, m6
	1021	psubw m6, m28, m14 ; 16
	1022	paddw m28, m14 ; 15
	1023	mova m14, m7
	1024	punpcklwd m7, m6, m4
	1025	punpckhwd m6, m4
	1026	punpckhwd m4, m17, m18
	1027	punpcklwd m17, m18
	1028	punpckhwd m18, m19, m20
	1029	punpcklwd m19, m20
	1030	punpckhwd m20, m21, m1
	1031	punpcklwd m21, m1
	1032	punpckhwd m1, m2, m11 ; i4 j4 i5 j5 i6 j6 i7 j7
	1033	punpcklwd m2, m11 ; i0 j1 i1 j1 i2 j2 i3 j3
	1034	punpckhwd m11, m12, m29 ; k4 l4 k5 l5 k6 l6 k7 l7
	1035	punpcklwd m12, m29 ; k0 l0 k1 l1 k2 l2 k3 l3
	1036	punpckhwd m29, m27, m26 ; m4 n4 m5 n5 m6 n6 m7 n7
	1037	punpcklwd m27, m26 ; m0 n0 m1 n1 m2 n2 m3 n3
	1038	punpckhwd m26, m16, m28 ; o4 p4 o5 p5 o6 p6 o7 p7
	1039	punpcklwd m16, m28 ; o0 p0 o1 p1 o2 p2 o3 p3
	1040	punpckhdq m28, m2, m12 ; i2 j2 k2 l2 i3 j3 k3 l3
	1041	punpckldq m2, m12 ; i0 j0 k0 l0 i1 j1 k1 l1
	1042	punpckhdq m12, m27, m16 ; m2 n2 o2 p2 m3 n3 o3 p3
	1043	punpckldq m27, m16 ; m0 n0 o0 p0 m1 n1 o1 p1
	1044	punpckhdq m16, m1, m11 ; i6 j6 k6 l6 i7 j7 k7 l7
	1045	punpckldq m1, m11 ; i4 j4 k4 l4 i5 j5 k5 l5
	1046	punpckhdq m11, m29, m26 ; m6 n6 o6 p6 m7 n7 o7 p7
	1047	punpckldq m29, m26 ; m4 n4 o4 p4 m5 n5 o5 p5
	1048	punpckhdq m26, m19, m21
	1049	punpckldq m19, m21
	1050	punpckhdq m21, m6, m4
	1051	punpckldq m6, m4
	1052	punpckhdq m4, m18, m20
	1053	punpckldq m18, m20
	1054	punpckhdq m20, m7, m17
	1055	punpckldq m7, m17
	1056	punpcklqdq m17, m28, m12 ; b02 b10 b18 b26
	1057	punpckhqdq m28, m12 ; b03 b11 b19 b27
	1058	punpckhqdq m12, m2, m27 ; b01 b09 b17 b25
	1059	punpcklqdq m2, m27 ; b00 b08 b16 b24
	1060	punpckhqdq m27, m1, m29 ; b05 b13 b21 b29
	1061	punpcklqdq m1, m29 ; b04 b12 b20 b28
	1062	punpckhqdq m29, m16, m11 ; b07 b15 b23 b31
	1063	punpcklqdq m16, m11 ; b06 b14 b22 b30
	1064	mova [rsp+64* 8], m12
	1065	mova [rsp+64* 9], m28
	1066	mova [rsp+64*10], m27
	1067	mova [rsp+64*11], m29
	1068	punpckhqdq m27, m20, m26 ; c03 c11 c19 c27
	1069	punpcklqdq m20, m26 ; c02 c10 c18 c26
	1070	punpckhqdq m26, m7, m19 ; c01 c09 c17 c25
	1071	punpcklqdq m7, m19 ; c00 c08 c16 c24
	1072	punpckhqdq m28, m6, m18 ; c05 c13 c21 c29
	1073	punpcklqdq m6, m18 ; c04 c12 c20 c28
	1074	punpckhqdq m29, m21, m4 ; c07 c15 c23 c31
	1075	punpcklqdq m21, m4 ; c06 c14 c22 c30
	1076	mov r3d, 64*28
	1077	pxor m4, m4
	1078	.zero_loop:
	1079	mova [cq+r3+64*0], m4
	1080	mova [cq+r3+64*1], m4
	1081	mova [cq+r3+64*2], m4
	1082	mova [cq+r3+64*3], m4
	1083	sub r3d, 64*4
	1084	jge .zero_loop
	1085	vshufi32x4 m4, m0, m2, q3232 ; a16 a24 b16 b24
	1086	vinserti32x8 m0, ym2, 1 ; a00 a08 b00 b08
	1087	vshufi32x4 m2, m7, m5, q3232 ; c16 c24 d16 d24
	1088	vinserti32x8 m7, ym5, 1 ; c00 c08 d00 d08
	1089	vshufi32x4 m5, m8, m1, q3232 ; a20 a28 b20 b28
	1090	vinserti32x8 m1, m8, ym1, 1 ; a04 a12 b04 b12
	1091	vshufi32x4 m8, m6, m3, q3232 ; c20 c28 d20 d28
	1092	vinserti32x8 m6, ym3, 1 ; c04 c12 d04 d12
	1093	vshufi32x4 m3, m1, m6, q3131 ; 12
	1094	vshufi32x4 m1, m6, q2020 ; 4
	1095	vshufi32x4 m6, m4, m2, q3131 ; 24
	1096	vshufi32x4 m4, m2, q2020 ; 16
	1097	vshufi32x4 m2, m0, m7, q3131 ; 8
	1098	vshufi32x4 m0, m7, q2020 ; 0
	1099	vshufi32x4 m7, m5, m8, q3131 ; 28
	1100	vshufi32x4 m5, m8, q2020 ; 20
	1101	vshufi32x4 m18, m14, m17, q3232 ; a18 a26 b18 b26
	1102	vinserti32x8 m14, ym17, 1 ; a02 a10 b02 b10
	1103	vshufi32x4 m17, m20, m13, q3232 ; c18 c26 d18 d26
	1104	vinserti32x8 m20, ym13, 1 ; c02 c10 d02 d10
	1105	vshufi32x4 m13, m21, m9, q3232 ; c22 c30 d22 d30
	1106	vinserti32x8 m21, ym9, 1 ; c06 c14 d06 d14
	1107	vshufi32x4 m19, m15, m16, q3232 ; a22 a30 b22 b30
	1108	vinserti32x8 m15, ym16, 1 ; a06 a14 b06 b14
	1109	vshufi32x4 m16, m14, m20, q3131 ; 10
	1110	vshufi32x4 m14, m20, q2020 ; 2
	1111	vshufi32x4 m20, m18, m17, q3131 ; 26
	1112	vshufi32x4 m18, m17, q2020 ; 18
	1113	vshufi32x4 m17, m15, m21, q3131 ; 14
	1114	vshufi32x4 m15, m21, q2020 ; 6
	1115	vshufi32x4 m21, m19, m13, q3131 ; 30
	1116	vshufi32x4 m19, m13, q2020 ; 22
	1117	call .idct16
	1118	mova [rsp+64*0], m14
	1119	mova [rsp+64*1], m15
	1120	mova [rsp+64*2], m16
	1121	mova [rsp+64*3], m17
	1122	mova [rsp+64*4], m18
	1123	mova [rsp+64*5], m19
	1124	mova [rsp+64*6], m20
	1125	mova [rsp+64*7], m21
	1126	mova m15, [rsp+64* 8]
	1127	mova m16, [rsp+64* 9]
	1128	mova m17, [rsp+64*10]
	1129	mova m19, [rsp+64*11]
	1130	mova m20, [rsp+64*12]
	1131	mova m21, [rsp+64*13]
	1132	mova m13, [rsp+64*14]
	1133	mova m18, [rsp+64*15]
	1134	vshufi32x4 m14, m22, m15, q3232 ; a17 a25 b17 b25
	1135	vinserti32x8 m22, ym15, 1 ; a01 a09 b01 b09
	1136	vshufi32x4 m15, m23, m16, q3232 ; a19 a27 b19 b27
	1137	vinserti32x8 m23, ym16, 1 ; a03 a11 b03 b11
	1138	vshufi32x4 m16, m24, m17, q3232 ; a21 a29 b21 b29
	1139	vinserti32x8 m24, ym17, 1 ; a05 a13 b05 b13
	1140	vshufi32x4 m17, m25, m19, q3232 ; a23 a31 b23 b31
	1141	vinserti32x8 m25, ym19, 1 ; a07 a15 b07 b15
	1142	vinserti32x8 m8, m26, ym20, 1 ; c01 c09 d01 d09
	1143	vshufi32x4 m26, m20, q3232 ; c17 c25 d17 d25
	1144	vinserti32x8 m9, m27, ym21, 1 ; c03 c11 d03 d11
	1145	vshufi32x4 m27, m21, q3232 ; c19 c27 d19 d27
	1146	vinserti32x8 m11, m28, ym13, 1 ; c05 c13 d05 d13
	1147	vshufi32x4 m28, m13, q3232 ; c21 c29 d21 d29
	1148	vinserti32x8 m12, m29, ym18, 1 ; c07 c15 d07 d15
	1149	vshufi32x4 m29, m18, q3232 ; c23 c31 d23 d31
	1150	vshufi32x4 m18, m14, m26, q3131 ; 25
	1151	vshufi32x4 m14, m26, q2020 ; 17
	1152	vshufi32x4 m19, m15, m27, q3131 ; 27
	1153	vshufi32x4 m15, m27, q2020 ; 19
	1154	vshufi32x4 m20, m16, m28, q3131 ; 29
	1155	vshufi32x4 m16, m28, q2020 ; 21
	1156	vshufi32x4 m21, m17, m29, q3131 ; 31
	1157	vshufi32x4 m17, m29, q2020 ; 23
	1158	vshufi32x4 m26, m22, m8, q3131 ; 9
	1159	vshufi32x4 m22, m8, q2020 ; 1
	1160	vshufi32x4 m27, m23, m9, q3131 ; 11
	1161	vshufi32x4 m23, m9, q2020 ; 3
	1162	vshufi32x4 m28, m24, m11, q3131 ; 13
	1163	vshufi32x4 m24, m11, q2020 ; 5
	1164	vshufi32x4 m29, m25, m12, q3131 ; 15
	1165	vshufi32x4 m25, m12, q2020 ; 7
	1166	call .main
	1167	jmp .end
	1168	.fast:
	1169	mova m14, [o(dup16_perm)]
	1170	pmovzxbw m9, [cq+64*0]
	1171	pmovzxbw m6, [cq+64*8]
	1172	vpermb m8, m14, [cq+64* 2]
	1173	vpermb m0, m14, [cq+64*14]
	1174	vpermb m5, m14, [cq+64*10]
	1175	vpermb m1, m14, [cq+64* 6]
	1176	vpermb m7, m14, [cq+64* 4]
	1177	vpermb m3, m14, [cq+64*12]
	1178	vpbroadcastd m10, [o(pd_8192)]
	1179	vpbroadcastq m13, [o(int_mshift)]
	1180	packuswb m9, m9
	1181	packuswb m6, m6
	1182	vpcmpub k7, m13, m10, 6
	1183	IDCT16_MAIN 1
	1184	vpermb m21, m14, [cq+64* 1]
	1185	vpermb m17, m14, [cq+64*15]
	1186	vpermb m20, m14, [cq+64* 9]
	1187	vpermb m15, m14, [cq+64* 7]
	1188	vpermb m18, m14, [cq+64* 5]
	1189	vpermb m16, m14, [cq+64*11]
	1190	vpermb m19, m14, [cq+64*13]
	1191	vpermb m14, m14, [cq+64* 3]
	1192	call .main_packed_fast
	1193	punpcklwd m8, m0, m2
	1194	punpckhwd m0, m2
	1195	punpcklwd m2, m1, m3
	1196	punpckhwd m1, m3
	1197	punpcklwd m3, m4, m6
	1198	punpckhwd m4, m6
	1199	punpcklwd m6, m5, m7
	1200	punpckhwd m5, m7
	1201	punpcklwd m7, m14, m16
	1202	punpckhwd m14, m16
	1203	punpcklwd m16, m15, m17
	1204	punpckhwd m15, m17
	1205	punpcklwd m17, m19, m21
	1206	punpckhwd m19, m21
	1207	punpckhwd m21, m18, m20
	1208	punpcklwd m18, m20
	1209	punpcklwd m20, m8, m1
	1210	punpckhwd m8, m1
	1211	punpcklwd m1, m0, m2
	1212	punpckhwd m0, m2
	1213	punpcklwd m2, m3, m5
	1214	punpckhwd m3, m5
	1215	punpcklwd m5, m4, m6
	1216	punpckhwd m4, m6
	1217	punpcklwd m6, m7, m15
	1218	punpckhwd m7, m15
	1219	punpcklwd m15, m14, m16
	1220	punpckhwd m14, m16
	1221	punpckhwd m16, m18, m19
	1222	punpcklwd m18, m19
	1223	punpcklwd m19, m21, m17
	1224	punpckhwd m21, m17
	1225	punpcklwd m17, m8, m0 ; a2 a6 aa ae
	1226	punpckhwd m8, m0 ; a3 a7 ab af
	1227	punpcklwd m0, m20, m1 ; a0 a4 a8 ac
	1228	punpckhwd m20, m1 ; a1 a5 a9 ad
	1229	punpcklwd m1, m2, m5 ; b0 b4 b8 bc
	1230	punpckhwd m2, m5 ; b1 b5 b9 bd
	1231	punpcklwd m5, m3, m4 ; b2 b6 ba be
	1232	punpckhwd m3, m4 ; b3 b7 bb bf
	1233	punpcklwd m4, m6, m15 ; c0 c4 c8 cc
	1234	punpckhwd m6, m15 ; c1 c5 c9 cd
	1235	punpcklwd m15, m7, m14 ; c2 c6 ca ce
	1236	punpckhwd m7, m14 ; c3 c7 cb cf
	1237	punpcklwd m14, m18, m19 ; d0 d4 d8 dc
	1238	punpckhwd m18, m19 ; d1 d5 d9 dd
	1239	punpcklwd m9, m16, m21 ; d2 d6 da de
	1240	punpckhwd m16, m21 ; d3 d7 db df
	1241	mov r3d, 64*12
	1242	pxor ym21, ym21
	1243	.fast_zero_loop:
	1244	mova [cq+r3+64*0], ym21
	1245	mova [cq+r3+64*1], ym21
	1246	mova [cq+r3+64*2], ym21
	1247	mova [cq+r3+64*3], ym21
	1248	sub r3d, 64*4
	1249	jge .fast_zero_loop
	1250	vshufi32x4 m21, m0, m1, q3232 ; a8 ac b8 bc
	1251	vinserti32x8 m0, ym1, 1 ; a0 a4 b0 b4
	1252	vinserti32x8 m1, m17, ym5, 1 ; a2 a6 b2 b6
	1253	vshufi32x4 m5, m17, m5, q3232 ; aa ae ba be
	1254	vinserti32x8 m17, m8, ym3, 1 ; a3 a7 b3 b7
	1255	vshufi32x4 m19, m8, m3, q3232 ; ab af bb bf
	1256	vinserti32x8 m3, m4, ym14, 1 ; c0 c4 d0 d4
	1257	vshufi32x4 m4, m14, q3232 ; c8 cc d8 dc
	1258	vinserti32x8 m14, m20, ym2, 1 ; a1 a5 b1 b5
	1259	vshufi32x4 m20, m2, q3232 ; a9 ad b9 bd
	1260	vinserti32x8 m2, m6, ym18, 1 ; c1 c5 d1 d5
	1261	vshufi32x4 m6, m18, q3232 ; c9 cd d9 dd
	1262	vinserti32x8 m18, m15, ym9, 1 ; c2 c6 d2 d6
	1263	vshufi32x4 m15, m9, q3232 ; ca ce da de
	1264	vinserti32x8 m9, m7, ym16, 1 ; c3 c7 d3 d7
	1265	vshufi32x4 m7, m16, q3232 ; cb cf db df
	1266	vshufi32x4 m22, m14, m2, q2020 ; 1
	1267	vshufi32x4 m24, m14, m2, q3131 ; 5
	1268	vshufi32x4 m23, m17, m9, q2020 ; 3
	1269	vshufi32x4 m25, m17, m9, q3131 ; 7
	1270	vshufi32x4 m16, m5, m15, q2020 ; 10
	1271	vshufi32x4 m17, m5, m15, q3131 ; 14
	1272	vshufi32x4 m14, m1, m18, q2020 ; 2
	1273	vshufi32x4 m15, m1, m18, q3131 ; 6
	1274	vshufi32x4 m1, m0, m3, q3131 ; 4
	1275	vshufi32x4 m0, m3, q2020 ; 0
	1276	vshufi32x4 m3, m21, m4, q3131 ; 12
	1277	vshufi32x4 m2, m21, m4, q2020 ; 8
	1278	vshufi32x4 m26, m20, m6, q2020 ; 9
	1279	vshufi32x4 m28, m20, m6, q3131 ; 13
	1280	vshufi32x4 m27, m19, m7, q2020 ; 11
	1281	vshufi32x4 m29, m19, m7, q3131 ; 15
	1282	call .idct16_fast
	1283	mova [rsp+64*0], m14
	1284	mova [rsp+64*1], m15
	1285	mova [rsp+64*2], m16
	1286	mova [rsp+64*3], m17
	1287	mova [rsp+64*4], m18
	1288	mova [rsp+64*5], m19
	1289	mova [rsp+64*6], m20
	1290	mova [rsp+64*7], m21
	1291	call .main_fast
	1292	.end:
	1293	lea r4, [strideq*3]
	1294	vpbroadcastd m12, [o(pw_512)]
	1295	movshdup m13, [o(itx_perm)]
	1296	lea r3, [dstq+r4*8]
	1297	lea r5, [strideq+r4] ; stride*4
	1298	add r3, r5 ; dst+stride*28
	1299	IDCT_32x32_END 29, 0, strideq*0, r4
	1300	IDCT_32x32_END 28, 1, strideq1, strideq2
	1301	IDCT_32x32_END 27, 2, strideq2, strideq1
	1302	IDCT_32x32_END 26, 3, r4 , strideq*0
	1303	IDCT_32x32_END 25, 4, strideq*0, r4
	1304	IDCT_32x32_END 24, 5, strideq1, strideq2
	1305	IDCT_32x32_END 23, 6, strideq2, strideq1
	1306	IDCT_32x32_END 22, 7, r4 , strideq*0
	1307	IDCT_32x32_END 21, 8, strideq*0, r4
	1308	IDCT_32x32_END 20, 9, strideq1, strideq2
	1309	IDCT_32x32_END 19, 10, strideq2, strideq1
	1310	IDCT_32x32_END 18, 11, r4 , strideq*0
	1311	IDCT_32x32_END 17, 12, strideq*0, r4
	1312	IDCT_32x32_END 16, 13, strideq1, strideq2
	1313	IDCT_32x32_END 15, 14, strideq2, strideq1
	1314	IDCT_32x32_END 14, 15, r4 , strideq*0
	1315	RET
	1316	ALIGN function_align
	1317	.idct16_fast:
	1318	vpbroadcastd m21, [o(pw_16305x2)]
	1319	vpbroadcastd m8, [o(pw_1606x2)]
	1320	vpbroadcastd m18, [o(pw_m10394x2)]
	1321	vpbroadcastd m9, [o(pw_12665x2)]
	1322	pmulhrsw m21, m14 ; t15a
	1323	vpbroadcastd m19, [o(pw_14449x2)]
	1324	pmulhrsw m14, m8 ; t8a
	1325	vpbroadcastd m8, [o(pw_7723x2)]
	1326	pmulhrsw m18, m17 ; t9a
	1327	vpbroadcastd m20, [o(pw_m4756x2)]
	1328	pmulhrsw m17, m9 ; t14a
	1329	vpbroadcastd m9, [o(pw_15679x2)]
	1330	pmulhrsw m19, m16 ; t13a
	1331	vpbroadcastd m5, [o(pw_m9102x2)]
	1332	pmulhrsw m16, m8 ; t10a
	1333	vpbroadcastd m8, [o(pw_13623x2)]
	1334	pmulhrsw m20, m15 ; t11a
	1335	vpbroadcastd m7, [o(pw_16069x2)]
	1336	pmulhrsw m15, m9 ; t12a
	1337	vpbroadcastd m9, [o(pw_3196x2)]
	1338	pmulhrsw m5, m3 ; t5a
	1339	vpbroadcastd m6, [o(pw_15137x2)]
	1340	pmulhrsw m3, m8 ; t6a
	1341	vpbroadcastd m8, [o(pw_6270x2)]
	1342	pmulhrsw m7, m1 ; t7a
	1343	vpbroadcastd m4, [o(pw_11585x2)]
	1344	pmulhrsw m1, m9 ; t4
	1345	vpbroadcastd m10, [o(pd_8192)]
	1346	pmulhrsw m6, m2 ; t3
	1347	pmulhrsw m2, m8 ; t2
	1348	pmulhrsw m4, m0 ; t0
	1349	mova m0, m4 ; t1
	1350	jmp .idct16b
	1351	ALIGN function_align
	1352	.idct16:
	1353	vpbroadcastd m10, [o(pd_8192)]
	1354	ITX_MULSUB_2W 14, 21, 8, 9, 10, 1606, 16305 ; t8a, t15a
	1355	ITX_MULSUB_2W 18, 17, 8, 9, 10, 12665, 10394 ; t9a, t14a
	1356	ITX_MULSUB_2W 16, 19, 8, 9, 10, 7723, 14449 ; t10a, t13a
	1357	ITX_MULSUB_2W 20, 15, 8, 9, 10, 15679, 4756 ; t11a, t12
	1358	ITX_MULSUB_2W 5, 3, 8, 9, 10, 13623, 9102 ; t5a, t6a
	1359	ITX_MULSUB_2W 1, 7, 8, 9, 10, 3196, 16069 ; t4a, t7a
	1360	ITX_MULSUB_2W 2, 6, 8, 9, 10, 6270, 15137 ; t2, t3
	1361	ITX_MULSUB_2W 0, 4, 8, 9, 10, 11585, 11585 ; t1, t0
	1362	.idct16b:
	1363	paddw m8, m20, m16 ; t11
	1364	psubw m20, m16 ; t10
	1365	paddw m16, m15, m19 ; t12
	1366	psubw m15, m19 ; t13
	1367	psubw m19, m14, m18 ; t9
	1368	paddw m14, m18 ; t8
	1369	psubw m18, m21, m17 ; t14
	1370	paddw m21, m17 ; t15
	1371	vpbroadcastd m11, [o(pw_6270_15137)]
	1372	vpbroadcastd m12, [o(pw_m15137_6270)]
	1373	ITX_MULSUB_2W 18, 19, 9, 17, 10, 11, 12 ; t9a, t14a
	1374	vpbroadcastd m11, [o(pw_m6270_m15137)]
	1375	ITX_MULSUB_2W 15, 20, 9, 17, 10, 12, 11 ; t10a, t13a
	1376	vpbroadcastd m11, [o(pw_11585_11585)]
	1377	vpbroadcastd m12, [o(pw_m11585_11585)]
	1378	paddw m9, m7, m3 ; t7
	1379	psubw m3, m7, m3 ; t6a
	1380	paddw m7, m1, m5 ; t4
	1381	psubw m1, m5 ; t5a
	1382	psubw m17, m14, m8 ; t11a
	1383	paddw m8, m14 ; t8a
	1384	paddw m14, m18, m15 ; t9
	1385	psubw m18, m15 ; t10
	1386	psubw m15, m19, m20 ; t13
	1387	paddw m19, m20 ; t14
	1388	paddw m20, m21, m16 ; t15a
	1389	psubw m16, m21, m16 ; t12a
	1390	ITX_MULSUB_2W 3, 1, 5, 21, 10, 11, 12 ; t5, t6
	1391	ITX_MULSUB_2W 15, 18, 5, 21, 10, 11, 12 ; t10a, t13a
	1392	ITX_MULSUB_2W 16, 17, 5, 21, 10, 11, 12 ; t11, t12
	1393	psubw m5, m0, m2 ; t2
	1394	paddw m2, m0 ; t1
	1395	paddw m0, m4, m6 ; t0
	1396	psubw m4, m6 ; t3
	1397	psubw m6, m2, m1 ; t6
	1398	paddw m1, m2 ; t1
	1399	paddw m2, m5, m3 ; t2
	1400	psubw m5, m3 ; t5
	1401	paddw m3, m4, m7 ; t3
	1402	psubw m4, m7 ; t4
	1403	psubw m7, m0, m9 ; t7
	1404	paddw m0, m9 ; t0
	1405	psubw m21, m0, m20 ; out15
	1406	paddw m0, m20 ; out0
	1407	psubw m20, m1, m19 ; out14
	1408	paddw m1, m19 ; out1
	1409	psubw m19, m2, m18 ; out13
	1410	paddw m2, m18 ; out2
	1411	psubw m18, m3, m17 ; out12
	1412	paddw m3, m17 ; out3
	1413	psubw m17, m4, m16 ; out11
	1414	paddw m4, m16 ; out4
	1415	psubw m16, m5, m15 ; out10
	1416	paddw m5, m15 ; out5
	1417	psubw m15, m6, m14 ; out9
	1418	paddw m6, m14 ; out6
	1419	psubw m14, m7, m8 ; out8
	1420	paddw m7, m8 ; out7
	1421	ret
	1422	ALIGN function_align
	1423	.main_fast:
	1424	vpbroadcastd m21, [o(pw_16364x2)]
	1425	vpbroadcastd m8, [o(pw_804x2)]
	1426	vpbroadcastd m14, [o(pw_m11003x2)]
	1427	vpbroadcastd m9, [o(pw_12140x2)]
	1428	pmulhrsw m21, m22 ; t31a
	1429	vpbroadcastd m17, [o(pw_14811x2)]
	1430	pmulhrsw m22, m8 ; t16a
	1431	vpbroadcastd m8, [o(pw_7005x2)]
	1432	pmulhrsw m14, m29 ; t30a
	1433	vpbroadcastd m18, [o(pw_m5520x2)]
	1434	pmulhrsw m29, m9 ; t17a
	1435	vpbroadcastd m9, [o(pw_15426x2)]
	1436	pmulhrsw m17, m26 ; t29a
	1437	vpbroadcastd m19, [o(pw_15893x2)]
	1438	pmulhrsw m26, m8 ; t18a
	1439	vpbroadcastd m8, [o(pw_3981x2)]
	1440	pmulhrsw m18, m25 ; t19a
	1441	vpbroadcastd m16, [o(pw_m8423x2)]
	1442	pmulhrsw m25, m9 ; t28a
	1443	vpbroadcastd m9, [o(pw_14053x2)]
	1444	pmulhrsw m19, m24 ; t27a
	1445	vpbroadcastd m15, [o(pw_13160x2)]
	1446	pmulhrsw m24, m8 ; t20a
	1447	vpbroadcastd m8, [o(pw_9760x2)]
	1448	pmulhrsw m16, m27 ; t21a
	1449	vpbroadcastd m20, [o(pw_m2404x2)]
	1450	pmulhrsw m27, m9 ; t26a
	1451	vpbroadcastd m9, [o(pw_16207x2)]
	1452	pmulhrsw m15, m28 ; t25a
	1453	pmulhrsw m28, m8 ; t22a
	1454	pmulhrsw m20, m23 ; t23a
	1455	pmulhrsw m23, m9 ; t24a
	1456	jmp .main2
	1457	ALIGN function_align
	1458	.main:
	1459	ITX_MULSUB_2W 22, 21, 8, 9, 10, 804, 16364 ; t16a, t31a
	1460	ITX_MULSUB_2W 14, 29, 8, 9, 10, 12140, 11003 ; t17a, t30a
	1461	ITX_MULSUB_2W 26, 17, 8, 9, 10, 7005, 14811 ; t18a, t29a
	1462	ITX_MULSUB_2W 18, 25, 8, 9, 10, 15426, 5520 ; t19a, t28a
	1463	ITX_MULSUB_2W 24, 19, 8, 9, 10, 3981, 15893 ; t20a, t27a
	1464	ITX_MULSUB_2W 16, 27, 8, 9, 10, 14053, 8423 ; t21a, t26a
	1465	ITX_MULSUB_2W 28, 15, 8, 9, 10, 9760, 13160 ; t22a, t25a
	1466	ITX_MULSUB_2W 20, 23, 8, 9, 10, 16207, 2404 ; t23a, t24a
	1467	.main2:
	1468	psubw m8, m22, m14 ; t17
	1469	paddw m22, m14 ; t16
	1470	paddw m14, m18, m26 ; t19
	1471	psubw m18, m26 ; t18
	1472	psubw m26, m24, m16 ; t21
	1473	paddw m24, m16 ; t20
	1474	psubw m16, m20, m28 ; t22
	1475	paddw m28, m20 ; t23
	1476	psubw m20, m23, m15 ; t25
	1477	paddw m23, m15 ; t24
	1478	psubw m15, m21, m29 ; t30
	1479	paddw m21, m29 ; t31
	1480	psubw m29, m19, m27 ; t26
	1481	paddw m19, m27 ; t27
	1482	paddw m27, m25, m17 ; t28
	1483	psubw m25, m17 ; t29
	1484	ITX_MULSUB_2W 15, 8, 9, 17, 10, 3196, 16069 ; t17a, t30a
	1485	ITX_MULSUB_2W 25, 18, 9, 17, 10, m16069, 3196 ; t18a, t29a
	1486	ITX_MULSUB_2W 29, 26, 9, 17, 10, 13623, 9102 ; t21a, t26a
	1487	ITX_MULSUB_2W 20, 16, 9, 17, 10, m9102, 13623 ; t22a, t25a
	1488	psubw m17, m21, m27 ; t28a
	1489	paddw m21, m27 ; t31a
	1490	psubw m27, m15, m25 ; t18
	1491	paddw m15, m25 ; t17
	1492	psubw m25, m20, m29 ; t21
	1493	paddw m20, m29 ; t22
	1494	psubw m29, m8, m18 ; t29
	1495	paddw m8, m18 ; t30
	1496	psubw m18, m22, m14 ; t19a
	1497	paddw m22, m14 ; t16a
	1498	psubw m14, m28, m24 ; t20a
	1499	paddw m24, m28 ; t23a
	1500	paddw m28, m16, m26 ; t25
	1501	psubw m16, m26 ; t26
	1502	psubw m26, m23, m19 ; t27a
	1503	paddw m23, m19 ; t24a
	1504	vpbroadcastd m12, [o(pw_m15137_6270)]
	1505	vpbroadcastd m11, [o(pw_6270_15137)]
	1506	ITX_MULSUB_2W 29, 27, 9, 19, 10, 11, 12 ; t18a, t29a
	1507	ITX_MULSUB_2W 17, 18, 9, 19, 10, 11, 12 ; t19, t28
	1508	vpbroadcastd m11, [o(pw_m6270_m15137)]
	1509	ITX_MULSUB_2W 16, 25, 9, 19, 10, 12, 11 ; t21a, t26a
	1510	ITX_MULSUB_2W 26, 14, 9, 19, 10, 12, 11 ; t20, t27
	1511	vpbroadcastd m12, [o(pw_m11585_11585)]
	1512	vpbroadcastd m11, [o(pw_11585_11585)]
	1513	psubw m19, m27, m25 ; t26
	1514	paddw m27, m25 ; t29
	1515	psubw m25, m17, m26 ; t20a
	1516	paddw m17, m26 ; t19a
	1517	paddw m26, m18, m14 ; t28a
	1518	psubw m18, m14 ; t27a
	1519	paddw m14, m22, m24 ; t16
	1520	psubw m22, m24 ; t23
	1521	psubw m24, m29, m16 ; t21
	1522	paddw m16, m29 ; t18
	1523	paddw m29, m21, m23 ; t31
	1524	psubw m21, m23 ; t24
	1525	psubw m23, m15, m20 ; t22a
	1526	paddw m15, m20 ; t17a
	1527	psubw m20, m8, m28 ; t25a
	1528	paddw m28, m8 ; t30a
	1529	ITX_MULSUB_2W 18, 25, 8, 9, 10, 11, 12 ; t20, t27
	1530	ITX_MULSUB_2W 19, 24, 8, 9, 10, 11, 12 ; t21a, t26a
	1531	ITX_MULSUB_2W 21, 22, 8, 9, 10, 11, 12 ; t23a, t24a
	1532	ITX_MULSUB_2W 20, 23, 8, 9, 10, 11, 12 ; t22, t25
	1533	ret
	1534	ALIGN function_align
	1535	.main_packed_fast:
	1536	vpbroadcastd m8, [o(pw_804_16364x2)]
	1537	vpbroadcastd m9, [o(pw_m11003_12140x2)]
	1538	vpbroadcastd m11, [o(pw_7005_14811x2)]
	1539	vpbroadcastd m12, [o(pw_m5520_15426x2)]
	1540	pmulhrsw m21, m8 ; t16a, t31a
	1541	vpbroadcastd m8, [o(pw_3981_15893x2)]
	1542	pmulhrsw m17, m9 ; t17a, t30a
	1543	vpbroadcastd m9, [o(pw_m8423_14053x2)]
	1544	pmulhrsw m20, m11 ; t18a, t29a
	1545	vpbroadcastd m11, [o(pw_9760_13160x2)]
	1546	pmulhrsw m15, m12 ; t19a, t28a
	1547	vpbroadcastd m12, [o(pw_m2404_16207x2)]
	1548	pmulhrsw m18, m8 ; t20a, t27a
	1549	pmulhrsw m16, m9 ; t21a, t26a
	1550	pmulhrsw m19, m11 ; t22a, t25a
	1551	pmulhrsw m14, m12 ; t23a, t24a
	1552	psubw m8, m21, m17 ; t17 t30
	1553	paddw m21, m17 ; t16 t31
	1554	psubw m17, m15, m20 ; t18 t29
	1555	paddw m20, m15 ; t19 t28
	1556	psubw m15, m18, m16 ; t21 t26
	1557	paddw m18, m16 ; t20 t27
	1558	psubw m16, m14, m19 ; t22 t25
	1559	paddw m14, m19 ; t23 t24
	1560	ITX_MUL2X_PACK 8, 9, 19, 10, 3196, 16069, 5 ; t17a t30a
	1561	ITX_MUL2X_PACK 17, 9, 19, 10, m16069, 3196, 5 ; t18a t29a
	1562	ITX_MUL2X_PACK 15, 9, 19, 10, 13623, 9102, 5 ; t21a t26a
	1563	ITX_MUL2X_PACK 16, 9, 19, 10, m9102, 13623, 5 ; t22a t25a
	1564	vpbroadcastd m11, [o(pw_m15137_6270)]
	1565	psubw m19, m21, m20 ; t19a t28a
	1566	paddw m21, m20 ; t16a t31a
	1567	psubw m20, m14, m18 ; t20a t27a
	1568	paddw m14, m18 ; t23a t24a
	1569	psubw m18, m8, m17 ; t18 t29
	1570	paddw m8, m17 ; t17 t30
	1571	psubw m17, m16, m15 ; t21 t26
	1572	paddw m15, m16 ; t22 t25
	1573	ITX_MUL2X_PACK 18, 9, 16, 10, 6270_15137, 11, 20 ; t18a t29a
	1574	ITX_MUL2X_PACK 19, 9, 16, 10, 6270_15137, 11, 20 ; t19 t28
	1575	ITX_MUL2X_PACK 20, 9, 16, 10, 11, m6270_m15137, 36 ; t20 t27
	1576	ITX_MUL2X_PACK 17, 9, 16, 10, 11, m6270_m15137, 36 ; t21a t26a
	1577	vbroadcasti32x4 m9, [o(deint_shuf)]
	1578	psubw m16, m21, m14 ; t23 t24
	1579	paddw m14, m21 ; t16 t31
	1580	psubw m21, m8, m15 ; t22a t25a
	1581	paddw m15, m8 ; t17a t30a
	1582	psubw m8, m18, m17 ; t21 t26
	1583	paddw m18, m17 ; t18 t29
	1584	paddw m17, m19, m20 ; t19a t28a
	1585	psubw m19, m20 ; t20a t27a
	1586	vpbroadcastd m11, [o(pw_m11585_11585)]
	1587	vpbroadcastd m12, [o(pw_11585_11585)]
	1588	REPX {pshufb x, m9}, m14, m15, m18, m17
	1589	mova m9, m10
	1590	vpdpwssd m9, m16, m11
	1591	mova m20, m10
	1592	vpdpwssd m20, m21, m11
	1593	psrad m9, 14
	1594	psrad m20, 14
	1595	packssdw m9, m20 ; t23a t22
	1596	mova m20, m10
	1597	vpdpwssd m20, m16, m12
	1598	mova m16, m10
	1599	vpdpwssd m16, m21, m12
	1600	psrad m20, 14
	1601	psrad m16, 14
	1602	packssdw m16, m20, m16 ; t24a t25
	1603	ITX_MUL2X_PACK 8, 21, 20, 10, 11, 12, 8 ; t21a t26a
	1604	ITX_MUL2X_PACK 19, 8, 11, 10, 11, 12, 8 ; t20 t27
	1605	packssdw m11, m20 ; t27 t26a
	1606	packssdw m8, m21 ; t20 t21a
	1607	punpcklqdq m20, m14, m15 ; t16 t17a
	1608	punpckhqdq m14, m15 ; t31 t30a
	1609	punpckhqdq m15, m17, m18 ; t28a t29
	1610	punpcklqdq m17, m18 ; t19a t18
	1611	psubw m21, m0, m14 ; out31 out30
	1612	paddw m0, m14 ; out0 out1
	1613	psubw m14, m7, m20 ; out16 out17
	1614	paddw m7, m20 ; out15 out14
	1615	psubw m20, m1, m15 ; out28 out29
	1616	paddw m1, m15 ; out3 out2
	1617	psubw m15, m6, m17 ; out19 out18
	1618	paddw m6, m17 ; out12 out13
	1619	psubw m17, m4, m9 ; out23 out22
	1620	paddw m4, m9 ; out8 out9
	1621	psubw m18, m3, m16 ; out24 out25
	1622	paddw m3, m16 ; out7 out6
	1623	psubw m16, m5, m8 ; out20 out21
	1624	paddw m5, m8 ; out11 out10
	1625	psubw m19, m2, m11 ; out27 out26
	1626	paddw m2, m11 ; out4 out5
	1627	ret
	1628
	1629	%endif

libavutil/mem_internal.h+2−0

@@ -131,4 +131,6 @@ · 130 unmodified lines
131		131
132	#define LOCAL_ALIGNED_32(t, v, ...) E1(LOCAL_ALIGNED_D(32, t, v, __VA_ARGS__,,))	132	#define LOCAL_ALIGNED_32(t, v, ...) E1(LOCAL_ALIGNED_D(32, t, v, __VA_ARGS__,,))
133		133
		134	#define LOCAL_ALIGNED_64(t, v, ...) E1(LOCAL_ALIGNED_D(64, t, v, __VA_ARGS__,,))
		135
134	#endif /* AVUTIL_MEM_INTERNAL_H */	136	#endif /* AVUTIL_MEM_INTERNAL_H */

tests/checkasm/vp9dsp.c+7−7

@@ -310,13 +310,13 @@ static int is_zero(const int16_t *c, int sz) · 309 unmodified lines
310		310
311	static void check_itxfm(void)	311	static void check_itxfm(void)
312	{	312	{
313	LOCAL_ALIGNED_32(uint8_t, src, [32 * 32 * 2]);	313	LOCAL_ALIGNED_64(uint8_t, src, [32 * 32 * 2]);
314	LOCAL_ALIGNED_32(uint8_t, dst, [32 * 32 * 2]);	314	LOCAL_ALIGNED_64(uint8_t, dst, [32 * 32 * 2]);
315	LOCAL_ALIGNED_32(uint8_t, dst0, [32 * 32 * 2]);	315	LOCAL_ALIGNED_64(uint8_t, dst0, [32 * 32 * 2]);
316	LOCAL_ALIGNED_32(uint8_t, dst1, [32 * 32 * 2]);	316	LOCAL_ALIGNED_64(uint8_t, dst1, [32 * 32 * 2]);
317	LOCAL_ALIGNED_32(int16_t, coef, [32 * 32 * 2]);	317	LOCAL_ALIGNED_64(int16_t, coef, [32 * 32 * 2]);
318	LOCAL_ALIGNED_32(int16_t, subcoef0, [32 * 32 * 2]);	318	LOCAL_ALIGNED_64(int16_t, subcoef0, [32 * 32 * 2]);
319	LOCAL_ALIGNED_32(int16_t, subcoef1, [32 * 32 * 2]);	319	LOCAL_ALIGNED_64(int16_t, subcoef1, [32 * 32 * 2]);
320	declare_func_emms(AV_CPU_FLAG_MMX \| AV_CPU_FLAG_MMXEXT, void, uint8_t dst, ptrdiff_t stride, int16_t block, int eob);	320	declare_func_emms(AV_CPU_FLAG_MMX \| AV_CPU_FLAG_MMXEXT, void, uint8_t dst, ptrdiff_t stride, int16_t block, int eob);
321	VP9DSPContext dsp;	321	VP9DSPContext dsp;
322	int y, x, tx, txtp, bit_depth, sub;	322	int y, x, tx, txtp, bit_depth, sub;

avcodec/x86/vp9: Add AVX-512ICL for 16x16 and 32x32 8bpc inverse transforms

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code