libavfilter/x86/vf_gblur: add localbuf and ff_horiz_slice_avx2/512()

We introduced a ff_horiz_slice_avx2/512() implemented on a new algorithm. In a nutshell, the new algorithm does three things, gathering data from 8/16 rows, blurring data, and scattering data back to the image buffer. Here we used a customized transpose 8x8/16x16 to avoid the huge overhead brought by gather and scatter instructions, which is dependent on the temporary buffer called localbuf added newly. Performance data: ff_horiz_slice_avx2(old): 109.89 ff_horiz_slice_avx2(new): 666.67 ff_horiz_slice_avx512: 1000

Wu Jianhua committed 5 years ago

Commit 4041c101 parent 0c54ab24 files · +601 −10Browse files

Unified · Split

4 files

libavfilter/gblur.h +3 −1
libavfilter/vf_gblur.c +9 −2
libavfilter/x86/vf_gblur.asm +575 −4
libavfilter/x86/vf_gblur_init.c +14 −3

libavfilter/gblur.h+3−1

@@ -39,9 +39,11 @@ typedef struct GBlurContext { · 38 unmodified lines
39	39
40	40	int flt;
41	41	int depth;
	42	+ int stride;
42	43	int planewidth[4];
43	44	int planeheight[4];
44	45	float *buffer;
	46	+ float *localbuf; ///< temporary buffer for horiz_slice. NULL if not used
45	47	float boundaryscale;
46	48	float boundaryscaleV;
47	49	float postscale;
@@ -49,7 +51,7 @@ typedef struct GBlurContext { · 1 unmodified line
49	51	float nu;
50	52	float nuV;
51	53	int nb_planes;
52		− void (horiz_slice)(float buffer, int width, int height, int steps, float nu, float bscale);
	54	+ void (horiz_slice)(float buffer, int width, int height, int steps, float nu, float bscale, float *localbuf);
53	55	void (verti_slice)(float buffer, int width, int height, int slice_start, int slice_end, int steps,
54	56	float nu, float bscale);
55	57	void (postscale_slice)(float buffer, int length, float postscale, float min, float max);

libavfilter/vf_gblur.c+9−2

@@ -64,7 +64,7 @@ static void postscale_c(float *buffer, int length, · 63 unmodified lines
64	64	}
65	65
66	66	static void horiz_slice_c(float *buffer, int width, int height, int steps,
67		− float nu, float bscale)
	67	+ float nu, float bscale, float *localbuf)
68	68	{
69	69	int step, x, y;
70	70	float *ptr;
@@ -97,9 +97,13 @@ static int filter_horizontally(AVFilterContext ctx, void arg, int jobnr, int n · 26 unmodified lines
97	97	const int steps = s->steps;
98	98	const float nu = s->nu;
99	99	float *buffer = s->buffer;
	100	+ float *localbuf = NULL;
	101	+
	102	+ if (s->localbuf)
	103	+ localbuf = s->localbuf + s->stride * width * slice_start;
100	104
101	105	s->horiz_slice(buffer + width * slice_start, width, slice_end - slice_start,
102		− steps, nu, boundaryscale);
	106	+ steps, nu, boundaryscale, localbuf);
103	107	emms_c();
104	108	return 0;
105	109	}
@@ -242,6 +246,7 @@ static int query_formats(AVFilterContext *ctx) · 136 unmodified lines
242	246
243	247	void ff_gblur_init(GBlurContext *s)
244	248	{
	249	+ s->localbuf = NULL;
245	250	s->horiz_slice = horiz_slice_c;
246	251	s->verti_slice = verti_slice_c;
247	252	s->postscale_slice = postscale_c;
@@ -384,6 +389,8 @@ static av_cold void uninit(AVFilterContext *ctx) · 136 unmodified lines
384	389	GBlurContext *s = ctx->priv;
385	390
386	391	av_freep(&s->buffer);
	392	+ if (s->localbuf)
	393	+ av_free(s->localbuf);
387	394	}
388	395
389	396	static const AVFilterPad gblur_inputs[] = {

libavfilter/x86/vf_gblur.asm+575−4

@@ -20,6 +20,14 @@ · 19 unmodified lines
20	20
21	21	%include "libavutil/x86/x86util.asm"
22	22
	23	+SECTION .data
	24	+
	25	+gblur_transpose_16x16_indices1: dq 2, 3, 0, 1, 6, 7, 4, 5
	26	+gblur_transpose_16x16_indices2: dq 1, 0, 3, 2, 5, 4, 7, 6
	27	+gblur_transpose_16x16_indices3: dd 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14
	28	+gblur_transpose_16x16_mask: dw 0xcc, 0x33, 0xaa, 0x55, 0xaaaa, 0x5555
	29	+gblur_vindex_width: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
	30	+
23	31	SECTION .text
24	32
25	33	%xdefine AVX2_MMSIZE 32
@@ -32,6 +40,29 @@ SECTION .text · 6 unmodified lines
32	40	%endrep
33	41	%endmacro
34	42
	43	+%macro KXNOR 2-*
	44	+%if mmsize == AVX512_MMSIZE
	45	+ kxnorw %2, %2, %2
	46	+%else
	47	+ %if %0 == 3
	48	+ mov %3, -1
	49	+ %else
	50	+ vpcmpeqd %1, %1, %1
	51	+ %endif
	52	+%endif
	53	+%endmacro
	54	+
	55	+%macro KMOVW 2-4
	56	+%if mmsize == AVX2_MMSIZE && %0 == 4
	57	+ mova %1, %2
	58	+%elif mmsize == AVX512_MMSIZE
	59	+ %if %0 == 4
	60	+ %rotate 2
	61	+ %endif
	62	+ kmovw %1, %2
	63	+%endif
	64	+%endmacro
	65	+
35	66	%macro PUSH_MASK 5
36	67	%if mmsize == AVX2_MMSIZE
37	68	%assign %%n mmsize/4
@@ -59,15 +90,546 @@ SECTION .text · 21 unmodified lines
59	90	%endif
60	91	%endmacro
61	92
62		−; void ff_horiz_slice_sse4(float *ptr, int width, int height, int steps,
63		−; float nu, float bscale)
	93	+%macro VGATHERDPS 4
	94	+%if mmsize == AVX2_MMSIZE
	95	+ vgatherdps %1, %2, %3
	96	+%else
	97	+ vgatherdps %1{%4}, %2
	98	+%endif
	99	+%endmacro
	100	+
	101	+%macro VSCATTERDPS128 7
	102	+ %rep 4
	103	+ mov %7, %6
	104	+ and %7, 1
	105	+ cmp %7, 0
	106	+ je %%end_scatter
	107	+ movss [%2 + %3*%4], xm%1
	108	+ vpshufd m%1, m%1, 0x39
	109	+ add %3, %5
	110	+ sar %6, 1
	111	+ %endrep
	112	+ %%end_scatter:
	113	+%endmacro
	114	+
	115	+; %1=register index
	116	+; %2=base address %3=vindex
	117	+; %4=scale %5=width
	118	+; %6=mask %7=tmp
	119	+; m15=reserved
	120	+%macro VSCATTERDPS256 7
	121	+ mova m15, m%1
	122	+ xor %3, %3
	123	+ VSCATTERDPS128 15, %2, %3, %4, %5, %6, %7
	124	+ vextractf128 xm15, m%1, 1
	125	+ VSCATTERDPS128 15, %2, %3, %4, %5, %6, %7
	126	+%endmacro
	127	+
	128	+; %1=base address %2=avx2 vindex
	129	+; %3=avx512 vindex %4=avx2 mask
	130	+; %5=avx512 mask %6=register index
	131	+; %7=width %8-*=tmp
	132	+%macro VSCATTERDPS 8-*
	133	+%if mmsize == AVX2_MMSIZE
	134	+ %if %0 == 9
	135	+ mov %9, %4
	136	+ VSCATTERDPS256 %6, %1, %2, 4, %7, %9, %8
	137	+ %else
	138	+ VSCATTERDPS256 %6, %1, %2, 4, %7, %4, %8
	139	+ %endif
	140	+%else
	141	+ vscatterdps [%1 + %3*4]{%5}, m%6
	142	+%endif
	143	+%endmacro
	144	+
	145	+%macro INIT_WORD_MASK 1-*
	146	+ %assign %%i 0
	147	+ %rep %0
	148	+ kmovw %1, [gblur_transpose_16x16_mask + %%i * 2]
	149	+ %assign %%i %%i+1
	150	+ %rotate 1
	151	+ %endrep
	152	+%endmacro
	153	+
	154	+%macro INIT_INDICES 1-*
	155	+ %assign %%i 1
	156	+ %rep %0
	157	+ movu %1, [gblur_transpose_16x16_indices %+ %%i]
	158	+ %assign %%i %%i+1
	159	+ %rotate 1
	160	+ %endrep
	161	+%endmacro
	162	+
	163	+%assign stack_offset 0
	164	+%macro PUSH_MM 1
	165	+%if mmsize == AVX2_MMSIZE
	166	+ movu [rsp + stack_offset], %1
	167	+ %assign stack_offset stack_offset+mmsize
	168	+%endif
	169	+%endmacro
	170	+
	171	+%macro POP_MM 1
	172	+%if mmsize == AVX2_MMSIZE
	173	+ %assign stack_offset stack_offset-mmsize
	174	+ movu %1, [rsp + stack_offset]
	175	+%endif
	176	+%endmacro
	177	+
	178	+%macro READ_LOCAL_BUFFER 1
	179	+ %if mmsize == AVX512_MMSIZE
	180	+ %assign %%i 19
	181	+ %else
	182	+ %assign %%i 9
	183	+ %endif
	184	+ %assign %%j %%i-1
	185	+ %assign %%k %1-1
	186	+ %xdefine %%m m %+ %%i
	187	+ mova %%m, m3
	188	+ FMULADD_PS %%m, %%m, m0, [localbufq + %%k * mmsize], %%m
	189	+ %assign %%k %%k-1
	190	+ %rep %1-1
	191	+ %xdefine %%m m %+ %%j
	192	+ mova %%m, m %+ %%i
	193	+ FMULADD_PS %%m, %%m, m0, [localbufq + %%k * mmsize], %%m
	194	+ %assign %%i %%i-1
	195	+ %assign %%j %%j-1
	196	+ %assign %%k %%k-1
	197	+ %endrep
	198	+ %if mmsize == AVX512_MMSIZE
	199	+ mova m3, m %+ %%i
	200	+ %endif
	201	+%endmacro
	202	+
	203	+%macro FMADD_WRITE 4
	204	+ FMULADD_PS %1, %1, %2, %3, %1
	205	+ mova %4, %1
	206	+%endmacro
	207	+
	208	+%macro WRITE_LOCAL_BUFFER_INTERNAL 8-16
	209	+ %assign %%i 0
	210	+ %rep %0
	211	+ FMADD_WRITE m3, m0, m %+ %1, [localbufq + %%i * mmsize]
	212	+ %assign %%i %%i+1
	213	+ %rotate 1
	214	+ %endrep
	215	+%endmacro
	216	+
	217	+%macro GATHERPS 1
	218	+ %if mmsize == AVX512_MMSIZE
	219	+ %assign %%i 4
	220	+ %else
	221	+ %assign %%i 2
	222	+ %endif
	223	+ movu m %+ %%i, [ptrq]
	224	+ mov strideq, widthq
	225	+ %assign %%i %%i+1
	226	+ %rep %1-2
	227	+ movu m %+ %%i, [ptrq + strideq*4]
	228	+ add strideq, widthq
	229	+ %assign %%i %%i+1
	230	+ %endrep
	231	+ movu m %+ %%i, [ptrq + strideq*4]
	232	+%endmacro
	233	+
	234	+%macro SCATTERPS_INTERNAL 8-16
	235	+ movu [ptrq + strideq*0], m %+ %1
	236	+ mov strideq, widthq
	237	+ %rotate 1
	238	+ %rep %0-2
	239	+ movu [ptrq + strideq*4], m %+ %1
	240	+ add strideq, widthq
	241	+ %rotate 1
	242	+ %endrep
	243	+ movu [ptrq + strideq*4], m %+ %1
	244	+%endmacro
	245	+
	246	+%macro BATCH_INSERT64X4 4-*
	247	+ %assign %%imm8 %1
	248	+ %rotate 1
	249	+ %rep (%0-1)/3
	250	+ vinserti64x4 m%1, m%2, ym%3, %%imm8
	251	+ %rotate 3
	252	+ %endrep
	253	+%endmacro
	254	+
	255	+%macro BATCH_EXTRACT_INSERT 2-*
	256	+ %assign %%imm8 %1
	257	+ %rotate 1
	258	+ %rep (%0-1)/2
	259	+ vextractf64x4 ym%1, m%1, %%imm8
	260	+ vextractf64x4 ym%2, m%2, %%imm8
	261	+ vinserti64x4 m%1, m%1, ym%2, %%imm8
	262	+ %rotate 2
	263	+ %endrep
	264	+%endmacro
	265	+
	266	+%macro BATCH_MOVE 2-*
	267	+ %rep %0/2
	268	+ mova m%1, m%2
	269	+ %rotate 2
	270	+ %endrep
	271	+%endmacro
	272	+
	273	+%macro BATCH_PERMUTE 3-*
	274	+ %xdefine %%decorator %1
	275	+ %xdefine %%mask %2
	276	+ %assign %%index %3
	277	+ %rotate 3
	278	+ %rep (%0-3)/2
	279	+ vperm %+ %%decorator m%1{%%mask}, m %+ %%index, m%2
	280	+ %rotate 2
	281	+ %endrep
	282	+%endmacro
64	283
	284	+; input : m3-m19
	285	+; output: m8 m5 m9 m15 m16 m7 m17 m27 m24 m21 m25 m19 m12 m23 m13 m11
	286	+%macro TRANSPOSE_16X16_AVX512 0
	287	+ BATCH_INSERT64X4 0x1, 20,4,12, 21,5,13, 22,6,14, 23,7,15
	288	+ BATCH_INSERT64X4 0x1, 24,8,16, 25,9,17, 26,10,18, 27,11,19
	289	+
	290	+ BATCH_EXTRACT_INSERT 0x1, 4,12, 5,13, 6,14, 7,15
	291	+ BATCH_EXTRACT_INSERT 0x1, 8,16, 9,17, 10,18, 11,19
	292	+
	293	+ BATCH_MOVE 12,20, 13,21, 14,22, 15,23
	294	+ BATCH_PERMUTE q, k6, 28, 12,24, 13,25, 14,26, 15,27
	295	+ BATCH_PERMUTE q, k5, 28, 24,20, 25,21, 26,22, 27,23
	296	+
	297	+ BATCH_MOVE 16,4, 17,5, 18,6, 19,7
	298	+ BATCH_PERMUTE q, k6, 28, 16,8, 17,9, 18,10, 19,11
	299	+ BATCH_PERMUTE q, k5, 28, 8,4, 9,5, 10,6, 11,7
	300	+
	301	+ BATCH_MOVE 4,12, 5,13, 6,24, 7,25
	302	+ BATCH_MOVE 20,16, 21,17, 22,8, 23,9
	303	+
	304	+ BATCH_PERMUTE q, k4, 29, 4,14, 5,15, 6,26, 7,27
	305	+ BATCH_PERMUTE q, k3, 29, 14,12, 15,13, 26,24, 27,25
	306	+ BATCH_PERMUTE q, k4, 29, 20,18, 21,19, 22,10, 23,11
	307	+ BATCH_PERMUTE q, k3, 29, 18,16, 19,17, 10,8, 11,9
	308	+
	309	+ BATCH_MOVE 8,4, 9,14, 16,6, 17,26
	310	+ BATCH_MOVE 24,20, 25,18, 12,22, 13,10
	311	+
	312	+ BATCH_PERMUTE d, k2, 30, 8,5, 9,15, 16,7, 17,27
	313	+ BATCH_PERMUTE d, k1, 30, 5,4, 15,14, 7,6, 27,26
	314	+ BATCH_PERMUTE d, k2, 30, 24,21, 25,19, 12,23, 13,11
	315	+ BATCH_PERMUTE d, k1, 30, 21,20, 19,18, 23,22, 11,10
	316	+%endmacro
	317	+
	318	+%macro INSERT_UNPACK 8
	319	+ vinsertf128 m%5, m%1, xm%3, 0x1
	320	+ vinsertf128 m%6, m%2, xm%4, 0x1
	321	+ vunpcklpd m%7, m%5, m%6
	322	+ vunpckhpd m%8, m%5, m%6
	323	+%endmacro
	324	+
	325	+%macro SHUFFLE 4
	326	+ vshufps m%3, m%1, m%2, 0x88
	327	+ vshufps m%4, m%1, m%2, 0xDD
	328	+ mova m%1, m%3
	329	+ mova m%2, m%4
	330	+%endmacro
	331	+
	332	+%macro EXTRACT_INSERT_UNPACK 6
	333	+ vextractf128 xm%1, m%1, 0x1
	334	+ vextractf128 xm%2, m%2, 0x1
	335	+ vinsertf128 m%3, m%3, xm%1, 0x0
	336	+ vinsertf128 m%4, m%4, xm%2, 0x0
	337	+ vunpcklpd m%5, m%3, m%4
	338	+ vunpckhpd m%6, m%3, m%4
	339	+%endmacro
	340	+
	341	+; Transpose 8x8 AVX2
	342	+; Limit the number ym# register to 16 for compatibility
	343	+; Used up registers instead of using stack memory
	344	+; Input: m2-m9
	345	+; Output: m12, m14, m13, m15, m8, m10, m9, m11
	346	+%macro TRANSPOSE_8X8_AVX2 0
	347	+ INSERT_UNPACK 2, 3, 6, 7, 10, 11, 12, 13
	348	+ INSERT_UNPACK 4, 5, 8, 9, 10, 11, 14, 15
	349	+
	350	+ SHUFFLE 12, 14, 10, 11
	351	+ SHUFFLE 13, 15, 10, 11
	352	+
	353	+ EXTRACT_INSERT_UNPACK 4, 5, 8, 9, 10, 11
	354	+ EXTRACT_INSERT_UNPACK 2, 3, 6, 7, 8, 9
	355	+
	356	+ SHUFFLE 8, 10, 6, 7
	357	+ SHUFFLE 9, 11, 6, 7
	358	+%endmacro
	359	+
	360	+%macro TRANSPOSE 0
	361	+ %if cpuflag(avx512)
	362	+ TRANSPOSE_16X16_AVX512
	363	+ %elif cpuflag(avx2)
	364	+ TRANSPOSE_8X8_AVX2
	365	+ %endif
	366	+%endmacro
	367	+
	368	+%macro WRITE_LOCAL_BUFFER 0
	369	+ %if cpuflag(avx512)
	370	+ WRITE_LOCAL_BUFFER_INTERNAL 8, 5, 9, 15, 16, 7, 17, 27, \
	371	+ 24, 21, 25, 19, 12, 23, 13, 11
	372	+ %elif cpuflag(avx2)
	373	+ WRITE_LOCAL_BUFFER_INTERNAL 12, 14, 13, 15, 8, 10, 9, 11
	374	+ %endif
	375	+%endmacro
	376	+
	377	+%macro SCATTERPS 0
	378	+ %if cpuflag(avx512)
	379	+ SCATTERPS_INTERNAL 8, 5, 9, 15, 16, 7, 17, 27, \
	380	+ 24, 21, 25, 19, 12, 23, 13, 11
	381	+ %elif cpuflag(avx2)
	382	+ SCATTERPS_INTERNAL 12, 14, 13, 15, 8, 10, 9, 11
	383	+ %endif
	384	+%endmacro
	385	+
	386	+%macro OPTIMIZED_LOOP_STEP 0
	387	+ lea stepd, [stepsd - 1]
	388	+ cmp stepd, 0
	389	+ jle %%bscale_scalar
	390	+%%loop_step:
	391	+ sub localbufq, mmsize
	392	+ mulps m3, m1
	393	+ movu [localbufq], m3
	394	+
	395	+ ; Filter leftwards
	396	+ lea xq, [widthq - 1]
	397	+ %%loop_step_x_back:
	398	+ sub localbufq, mmsize
	399	+ FMULADD_PS m3, m3, m0, [localbufq], m3
	400	+ movu [localbufq], m3
	401	+
	402	+ dec xq
	403	+ cmp xq, 0
	404	+ jg %%loop_step_x_back
	405	+
	406	+ ; Filter rightwards
	407	+ mulps m3, m1
	408	+ movu [localbufq], m3
	409	+ add localbufq, mmsize
	410	+
	411	+ lea xq, [widthq - 1]
	412	+ %%loop_step_x:
	413	+ FMULADD_PS m3, m3, m0, [localbufq], m3
	414	+ movu [localbufq], m3
	415	+ add localbufq, mmsize
	416	+
	417	+ dec xq
	418	+ cmp xq, 0
	419	+ jg %%loop_step_x
	420	+
	421	+ dec stepd
	422	+ cmp stepd, 0
	423	+ jg %%loop_step
	424	+
	425	+%%bscale_scalar:
	426	+%endmacro
	427	+
	428	+;***************************************************************************
	429	+; void ff_horiz_slice(float *ptr, int width, int height, int steps,
	430	+; float nu, float bscale)
	431	+;***************************************************************************
65	432	%macro HORIZ_SLICE 0
66	433	%if UNIX64
	434	+%if cpuflag(avx512) \|\| cpuflag(avx2)
	435	+cglobal horiz_slice, 5, 12, mmnum, 0-mmsize*4, buffer, width, height, steps, \
	436	+ localbuf, x, y, step, stride, remain, ptr, mask
	437	+%else
67	438	cglobal horiz_slice, 4, 9, 9, ptr, width, height, steps, x, y, step, stride, remain
	439	+%endif
	440	+%else
	441	+%if cpuflag(avx512) \|\| cpuflag(avx2)
	442	+cglobal horiz_slice, 5, 12, mmnum, 0-mmsize*4, buffer, width, height, steps, nu, bscale, \
	443	+ localbuf, x, y, step, stride, remain, ptr, mask
68	444	%else
69	445	cglobal horiz_slice, 4, 9, 9, ptr, width, height, steps, nu, bscale, x, y, step, stride, remain
70	446	%endif
	447	+%endif
	448	+%if cpuflag(avx512) \|\| cpuflag(avx2)
	449	+%assign rows mmsize/4
	450	+%assign cols mmsize/4
	451	+%if WIN64
	452	+ VBROADCASTSS m0, num ; nu
	453	+ VBROADCASTSS m1, bscalem ; bscale
	454	+
	455	+ mov nuq, localbufm
	456	+ DEFINE_ARGS buffer, width, height, steps, \
	457	+ localbuf, x, y, step, stride, remain, ptr, mask
	458	+ MOVSXDIFNIDN width, height, steps
	459	+%else
	460	+ VBROADCASTSS m0, xmm0 ; nu
	461	+ VBROADCASTSS m1, xmm1 ; bscale
	462	+%endif
	463	+
	464	+%if cpuflag(avx512)
	465	+ vpbroadcastd m2, widthd
	466	+ INIT_WORD_MASK k6, k5, k4, k3, k2, k1
	467	+ INIT_INDICES m28, m29, m30
	468	+%else
	469	+ movd xm2, widthd
	470	+ VBROADCASTSS m2, xm2
	471	+%endif
	472	+
	473	+ vpmulld m2, m2, [gblur_vindex_width] ; vindex width
	474	+
	475	+ xor yq, yq ; y = 0
	476	+ xor xq, xq ; x = 0
	477	+
	478	+ cmp heightq, rows
	479	+ jl .y_scalar
	480	+ sub heightq, rows
	481	+
	482	+.loop_y:
	483	+ ; ptr = buffer + y * width;
	484	+ mov ptrq, yq
	485	+ imul ptrq, widthq
	486	+ lea ptrq, [bufferq + ptrq*4]
	487	+
	488	+ KXNOR m5, k7
	489	+ VGATHERDPS m3, [ptrq + m2*4], m5, k7
	490	+ mulps m3, m1
	491	+ movu [localbufq], m3
	492	+ add ptrq, 4
	493	+ add localbufq, mmsize
	494	+
	495	+ ; Filter rightwards
	496	+ PUSH_MM m2
	497	+ lea xq, [widthq - 1]
	498	+ .loop_x:
	499	+ PUSH_MM m3
	500	+ GATHERPS cols
	501	+ TRANSPOSE
	502	+ POP_MM m3
	503	+ WRITE_LOCAL_BUFFER
	504	+
	505	+ add ptrq, mmsize
	506	+ add localbufq, rows * mmsize
	507	+ sub xq, cols
	508	+ cmp xq, cols
	509	+ jge .loop_x
	510	+ POP_MM m2
	511	+
	512	+ cmp xq, 0
	513	+ jle .bscale_scalar
	514	+ .loop_x_scalar:
	515	+ KXNOR m5, k7
	516	+ VGATHERDPS m4, [ptrq + m2*4], m5, k7
	517	+ FMULADD_PS m3, m3, m0, m4, m3
	518	+ movu [localbufq], m3
	519	+
	520	+ add ptrq, 0x4
	521	+ add localbufq, mmsize
	522	+ dec xq
	523	+ cmp xq, 0
	524	+ jg .loop_x_scalar
	525	+
	526	+ OPTIMIZED_LOOP_STEP
	527	+
	528	+ .bscale_scalar:
	529	+ sub ptrq, 4
	530	+ sub localbufq, mmsize
	531	+ mulps m3, m1
	532	+ KXNOR m5, k7, maskq
	533	+ VSCATTERDPS ptrq, strideq, m2, maskq, k7, 3, widthq, remainq
	534	+
	535	+ ; Filter leftwards
	536	+ PUSH_MM m2
	537	+ lea xq, [widthq - 1]
	538	+ .loop_x_back:
	539	+ sub localbufq, rows * mmsize
	540	+ READ_LOCAL_BUFFER cols
	541	+ PUSH_MM m2
	542	+ TRANSPOSE
	543	+ POP_MM m3
	544	+ sub ptrq, mmsize
	545	+ SCATTERPS
	546	+
	547	+ sub xq, cols
	548	+ cmp xq, cols
	549	+ jge .loop_x_back
	550	+ POP_MM m2
	551	+
	552	+ cmp xq, 0
	553	+ jle .end_loop_x
	554	+ .loop_x_back_scalar:
	555	+ sub ptrq, 0x4
	556	+ sub localbufq, mmsize
	557	+ FMULADD_PS m3, m3, m0, [localbufq], m3
	558	+ KXNOR m5, k7, maskq
	559	+ VSCATTERDPS ptrq, strideq, m2, maskq, k7, 3, widthq, remainq
	560	+
	561	+ dec xq
	562	+ cmp xq, 0
	563	+ jg .loop_x_back_scalar
	564	+
	565	+ .end_loop_x:
	566	+
	567	+ add yq, rows
	568	+ cmp yq, heightq
	569	+ jle .loop_y
	570	+
	571	+ add heightq, rows
	572	+ cmp yq, heightq
	573	+ jge .end_scalar
	574	+
	575	+ mov remainq, widthq
	576	+ imul remainq, mmsize
	577	+ add ptrq, remainq
	578	+
	579	+.y_scalar:
	580	+ mov remainq, heightq
	581	+ sub remainq, yq
	582	+ mov maskq, 1
	583	+ shlx maskq, maskq, remainq
	584	+ sub maskq, 1
	585	+ mov remainq, maskq
	586	+ PUSH_MASK m5, k1, remaind, xd, rsp + 0x20
	587	+
	588	+ mov ptrq, yq
	589	+ imul ptrq, widthq
	590	+ lea ptrq, [bufferq + ptrq * 4] ; ptrq = buffer + y * width
	591	+ KMOVW m6, m5, k7, k1
	592	+ VGATHERDPS m3, [ptrq + m2 * 4], m6, k7
	593	+ mulps m3, m1 ; p0 *= bscale
	594	+ movu [localbufq], m3
	595	+ add localbufq, mmsize
	596	+
	597	+ ; Filter rightwards
	598	+ lea xq, [widthq - 1]
	599	+ .y_scalar_loop_x:
	600	+ add ptrq, 4
	601	+ KMOVW m6, m5, k7, k1
	602	+ VGATHERDPS m4, [ptrq + m2 * 4], m6, k7
	603	+ FMULADD_PS m3, m3, m0, m4, m3
	604	+ movu [localbufq], m3
	605	+ add localbufq, mmsize
	606	+
	607	+ dec xq
	608	+ cmp xq, 0
	609	+ jg .y_scalar_loop_x
	610	+
	611	+ OPTIMIZED_LOOP_STEP
	612	+
	613	+ sub localbufq, mmsize
	614	+ mulps m3, m1 ; p0 *= bscale
	615	+ KMOVW k7, k1
	616	+ VSCATTERDPS ptrq, strideq, m2, maskq, k7, 3, widthq, remainq, heightq
	617	+
	618	+ ; Filter leftwards
	619	+ lea xq, [widthq - 1]
	620	+ .y_scalar_loop_x_back:
	621	+ sub ptrq, 4
	622	+ sub localbufq, mmsize
	623	+ FMULADD_PS m3, m3, m0, [localbufq], m3
	624	+ KMOVW k7, k1
	625	+ VSCATTERDPS ptrq, strideq, m2, maskq, k7, 3, widthq, remainq, heightq
	626	+ dec xq
	627	+ cmp xq, 0
	628	+ jg .y_scalar_loop_x_back
	629	+
	630	+.end_scalar:
	631	+ RET
	632	+%else
71	633	%if WIN64
72	634	movss m0, num
73	635	movss m1, bscalem
@@ -211,16 +773,26 @@ cglobal horiz_slice, 4, 9, 9, ptr, width, height, steps, nu, bscale, x, y, step, · 137 unmodified lines
211	773	jl .loop_y
212	774
213	775	RET
	776	+%endif
214	777	%endmacro
215	778
216	779	%if ARCH_X86_64
217	780	INIT_XMM sse4
218	781	HORIZ_SLICE
219	782
220		−INIT_XMM avx2
	783	+%if HAVE_AVX2_EXTERNAL
	784	+INIT_YMM avx2
	785	+%xdefine mmnum 16
221	786	HORIZ_SLICE
222	787	%endif
223	788
	789	+%if HAVE_AVX512_EXTERNAL
	790	+INIT_ZMM avx512
	791	+%xdefine mmnum 32
	792	+HORIZ_SLICE
	793	+%endif
	794	+%endif
	795	+
224	796	%macro POSTSCALE_SLICE 0
225	797	cglobal postscale_slice, 2, 2, 4, ptr, length, postscale, min, max
226	798	shl lengthd, 2
@@ -270,7 +842,6 @@ INIT_ZMM avx512 · 43 unmodified lines
270	842	POSTSCALE_SLICE
271	843	%endif
272	844
273		−
274	845	;*******************************************************************************
275	846	; void ff_verti_slice(float *buffer, int width, int height, int column_begin,
276	847	; int column_end, int steps, float nu, float bscale);

libavfilter/x86/vf_gblur_init.c+14−3

@@ -24,8 +24,9 @@ · 23 unmodified lines
24	24	#include "libavutil/x86/cpu.h"
25	25	#include "libavfilter/gblur.h"
26	26
27		−void ff_horiz_slice_sse4(float *ptr, int width, int height, int steps, float nu, float bscale);
28		−void ff_horiz_slice_avx2(float *ptr, int width, int height, int steps, float nu, float bscale);
	27	+void ff_horiz_slice_sse4(float ptr, int width, int height, int steps, float nu, float bscale, float localbuf);
	28	+void ff_horiz_slice_avx2(float ptr, int width, int height, int steps, float nu, float bscale, float localbuf);
	29	+void ff_horiz_slice_avx512(float ptr, int width, int height, int steps, float nu, float bscale, float localbuf);
29	30
30	31	void ff_postscale_slice_sse(float *ptr, int length, float postscale, float min, float max);
31	32	void ff_postscale_slice_avx2(float *ptr, int length, float postscale, float min, float max);
@@ -51,12 +52,22 @@ av_cold void ff_gblur_init_x86(GBlurContext *s) · 19 unmodified lines
51	52	s->horiz_slice = ff_horiz_slice_sse4;
52	53	}
53	54	if (EXTERNAL_AVX2(cpu_flags)) {
54		− s->horiz_slice = ff_horiz_slice_avx2;
55	55	s->verti_slice = ff_verti_slice_avx2;
56	56	}
57	57	if (EXTERNAL_AVX512(cpu_flags)) {
58	58	s->postscale_slice = ff_postscale_slice_avx512;
59	59	s->verti_slice = ff_verti_slice_avx512;
60	60	}
	61	+ if (EXTERNAL_AVX2(cpu_flags)) {
	62	+ s->stride = EXTERNAL_AVX512(cpu_flags) ? 16 : 8;
	63	+ s->localbuf = av_malloc(s->stride * sizeof(float) * s->planewidth[0] * s->planeheight[0]);
	64	+ if (!s->localbuf)
	65	+ return;
	66	+
	67	+ s->horiz_slice = ff_horiz_slice_avx2;
	68	+ if (EXTERNAL_AVX512(cpu_flags)) {
	69	+ s->horiz_slice = ff_horiz_slice_avx512;
	70	+ }
	71	+ }
61	72	#endif
62	73	}

libavfilter/x86/vf_gblur: add localbuf and ff_horiz_slice_avx2/512()

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code