History

libavfilter/x86/vf_gblur: add localbuf and ff_horiz_slice_avx2/512()

We introduced a ff_horiz_slice_avx2/512() implemented on a new algorithm. In a nutshell, the new algorithm does three things, gathering data from 8/16 rows, blurring data, and scattering data back to the image buffer. Here we used a customized transpose 8x8/16x16 to avoid the huge overhead brought by gather and scatter instructions, which is dependent on the temporary buffer called localbuf added newly. Performance data: ff_horiz_slice_avx2(old): 109.89 ff_horiz_slice_avx2(new): 666.67 ff_horiz_slice_avx512: 1000

Wu Jianhua committed 5 years ago

Commit 4041c101 parent 0c54ab24 files · +601 −10Browse files

Showing the changes to libavfilter (4 of 4 files). Show all

Unified · Split

4 files
  1. libavfilter/gblur.h +3 −1
  2. libavfilter/vf_gblur.c +9 −2
  3. libavfilter/x86/vf_gblur.asm +575 −4
  4. libavfilter/x86/vf_gblur_init.c +14 −3
libavfilter/gblur.h+3−1
@@ -39,9 +39,11 @@ typedef struct GBlurContext { · 38 unmodified lines
39 39
40 int flt;40 int flt;
41 int depth;41 int depth;
42 int stride;
42 int planewidth[4];43 int planewidth[4];
43 int planeheight[4];44 int planeheight[4];
44 float *buffer;45 float *buffer;
46 float *localbuf; ///< temporary buffer for horiz_slice. NULL if not used
45 float boundaryscale;47 float boundaryscale;
46 float boundaryscaleV;48 float boundaryscaleV;
47 float postscale;49 float postscale;
@@ -49,7 +51,7 @@ typedef struct GBlurContext { · 1 unmodified line
49 float nu;51 float nu;
50 float nuV;52 float nuV;
51 int nb_planes;53 int nb_planes;
52 void (*horiz_slice)(float *buffer, int width, int height, int steps, float nu, float bscale);54 void (*horiz_slice)(float *buffer, int width, int height, int steps, float nu, float bscale, float *localbuf);
53 void (*verti_slice)(float *buffer, int width, int height, int slice_start, int slice_end, int steps,55 void (*verti_slice)(float *buffer, int width, int height, int slice_start, int slice_end, int steps,
54 float nu, float bscale);56 float nu, float bscale);
55 void (*postscale_slice)(float *buffer, int length, float postscale, float min, float max);57 void (*postscale_slice)(float *buffer, int length, float postscale, float min, float max);
libavfilter/vf_gblur.c+9−2
@@ -64,7 +64,7 @@ static void postscale_c(float *buffer, int length, · 63 unmodified lines
64}64}
65 65
66static void horiz_slice_c(float *buffer, int width, int height, int steps,66static void horiz_slice_c(float *buffer, int width, int height, int steps,
67 float nu, float bscale)67 float nu, float bscale, float *localbuf)
68{68{
69 int step, x, y;69 int step, x, y;
70 float *ptr;70 float *ptr;
@@ -97,9 +97,13 @@ static int filter_horizontally(AVFilterContext *ctx, void *arg, int jobnr, int n · 26 unmodified lines
97 const int steps = s->steps;97 const int steps = s->steps;
98 const float nu = s->nu;98 const float nu = s->nu;
99 float *buffer = s->buffer;99 float *buffer = s->buffer;
100 float *localbuf = NULL;
101
102 if (s->localbuf)
103 localbuf = s->localbuf + s->stride * width * slice_start;
100 104
101 s->horiz_slice(buffer + width * slice_start, width, slice_end - slice_start,105 s->horiz_slice(buffer + width * slice_start, width, slice_end - slice_start,
102 steps, nu, boundaryscale);106 steps, nu, boundaryscale, localbuf);
103 emms_c();107 emms_c();
104 return 0;108 return 0;
105}109}
@@ -242,6 +246,7 @@ static int query_formats(AVFilterContext *ctx) · 136 unmodified lines
242 246
243void ff_gblur_init(GBlurContext *s)247void ff_gblur_init(GBlurContext *s)
244{248{
249 s->localbuf = NULL;
245 s->horiz_slice = horiz_slice_c;250 s->horiz_slice = horiz_slice_c;
246 s->verti_slice = verti_slice_c;251 s->verti_slice = verti_slice_c;
247 s->postscale_slice = postscale_c;252 s->postscale_slice = postscale_c;
@@ -384,6 +389,8 @@ static av_cold void uninit(AVFilterContext *ctx) · 136 unmodified lines
384 GBlurContext *s = ctx->priv;389 GBlurContext *s = ctx->priv;
385 390
386 av_freep(&s->buffer);391 av_freep(&s->buffer);
392 if (s->localbuf)
393 av_free(s->localbuf);
387}394}
388 395
389static const AVFilterPad gblur_inputs[] = {396static const AVFilterPad gblur_inputs[] = {
libavfilter/x86/vf_gblur.asm+575−4
@@ -20,6 +20,14 @@ · 19 unmodified lines
20 20
21%include "libavutil/x86/x86util.asm"21%include "libavutil/x86/x86util.asm"
22 22
23SECTION .data
24
25gblur_transpose_16x16_indices1: dq 2, 3, 0, 1, 6, 7, 4, 5
26gblur_transpose_16x16_indices2: dq 1, 0, 3, 2, 5, 4, 7, 6
27gblur_transpose_16x16_indices3: dd 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14
28gblur_transpose_16x16_mask: dw 0xcc, 0x33, 0xaa, 0x55, 0xaaaa, 0x5555
29gblur_vindex_width: dd 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
30
23SECTION .text31SECTION .text
24 32
25%xdefine AVX2_MMSIZE 3233%xdefine AVX2_MMSIZE 32
@@ -32,6 +40,29 @@ SECTION .text · 6 unmodified lines
32 %endrep40 %endrep
33%endmacro41%endmacro
34 42
43%macro KXNOR 2-*
44%if mmsize == AVX512_MMSIZE
45 kxnorw %2, %2, %2
46%else
47 %if %0 == 3
48 mov %3, -1
49 %else
50 vpcmpeqd %1, %1, %1
51 %endif
52%endif
53%endmacro
54
55%macro KMOVW 2-4
56%if mmsize == AVX2_MMSIZE && %0 == 4
57 mova %1, %2
58%elif mmsize == AVX512_MMSIZE
59 %if %0 == 4
60 %rotate 2
61 %endif
62 kmovw %1, %2
63%endif
64%endmacro
65
35%macro PUSH_MASK 566%macro PUSH_MASK 5
36%if mmsize == AVX2_MMSIZE67%if mmsize == AVX2_MMSIZE
37 %assign %%n mmsize/468 %assign %%n mmsize/4
@@ -59,15 +90,546 @@ SECTION .text · 21 unmodified lines
59%endif90%endif
60%endmacro91%endmacro
61 92
62; void ff_horiz_slice_sse4(float *ptr, int width, int height, int steps,93%macro VGATHERDPS 4
63; float nu, float bscale)94%if mmsize == AVX2_MMSIZE
95 vgatherdps %1, %2, %3
96%else
97 vgatherdps %1{%4}, %2
98%endif
99%endmacro
100
101%macro VSCATTERDPS128 7
102 %rep 4
103 mov %7, %6
104 and %7, 1
105 cmp %7, 0
106 je %%end_scatter
107 movss [%2 + %3*%4], xm%1
108 vpshufd m%1, m%1, 0x39
109 add %3, %5
110 sar %6, 1
111 %endrep
112 %%end_scatter:
113%endmacro
114
115; %1=register index
116; %2=base address %3=vindex
117; %4=scale %5=width
118; %6=mask %7=tmp
119; m15=reserved
120%macro VSCATTERDPS256 7
121 mova m15, m%1
122 xor %3, %3
123 VSCATTERDPS128 15, %2, %3, %4, %5, %6, %7
124 vextractf128 xm15, m%1, 1
125 VSCATTERDPS128 15, %2, %3, %4, %5, %6, %7
126%endmacro
127
128; %1=base address %2=avx2 vindex
129; %3=avx512 vindex %4=avx2 mask
130; %5=avx512 mask %6=register index
131; %7=width %8-*=tmp
132%macro VSCATTERDPS 8-*
133%if mmsize == AVX2_MMSIZE
134 %if %0 == 9
135 mov %9, %4
136 VSCATTERDPS256 %6, %1, %2, 4, %7, %9, %8
137 %else
138 VSCATTERDPS256 %6, %1, %2, 4, %7, %4, %8
139 %endif
140%else
141 vscatterdps [%1 + %3*4]{%5}, m%6
142%endif
143%endmacro
144
145%macro INIT_WORD_MASK 1-*
146 %assign %%i 0
147 %rep %0
148 kmovw %1, [gblur_transpose_16x16_mask + %%i * 2]
149 %assign %%i %%i+1
150 %rotate 1
151 %endrep
152%endmacro
153
154%macro INIT_INDICES 1-*
155 %assign %%i 1
156 %rep %0
157 movu %1, [gblur_transpose_16x16_indices %+ %%i]
158 %assign %%i %%i+1
159 %rotate 1
160 %endrep
161%endmacro
162
163%assign stack_offset 0
164%macro PUSH_MM 1
165%if mmsize == AVX2_MMSIZE
166 movu [rsp + stack_offset], %1
167 %assign stack_offset stack_offset+mmsize
168%endif
169%endmacro
170
171%macro POP_MM 1
172%if mmsize == AVX2_MMSIZE
173 %assign stack_offset stack_offset-mmsize
174 movu %1, [rsp + stack_offset]
175%endif
176%endmacro
177
178%macro READ_LOCAL_BUFFER 1
179 %if mmsize == AVX512_MMSIZE
180 %assign %%i 19
181 %else
182 %assign %%i 9
183 %endif
184 %assign %%j %%i-1
185 %assign %%k %1-1
186 %xdefine %%m m %+ %%i
187 mova %%m, m3
188 FMULADD_PS %%m, %%m, m0, [localbufq + %%k * mmsize], %%m
189 %assign %%k %%k-1
190 %rep %1-1
191 %xdefine %%m m %+ %%j
192 mova %%m, m %+ %%i
193 FMULADD_PS %%m, %%m, m0, [localbufq + %%k * mmsize], %%m
194 %assign %%i %%i-1
195 %assign %%j %%j-1
196 %assign %%k %%k-1
197 %endrep
198 %if mmsize == AVX512_MMSIZE
199 mova m3, m %+ %%i
200 %endif
201%endmacro
202
203%macro FMADD_WRITE 4
204 FMULADD_PS %1, %1, %2, %3, %1
205 mova %4, %1
206%endmacro
207
208%macro WRITE_LOCAL_BUFFER_INTERNAL 8-16
209 %assign %%i 0
210 %rep %0
211 FMADD_WRITE m3, m0, m %+ %1, [localbufq + %%i * mmsize]
212 %assign %%i %%i+1
213 %rotate 1
214 %endrep
215%endmacro
216
217%macro GATHERPS 1
218 %if mmsize == AVX512_MMSIZE
219 %assign %%i 4
220 %else
221 %assign %%i 2
222 %endif
223 movu m %+ %%i, [ptrq]
224 mov strideq, widthq
225 %assign %%i %%i+1
226 %rep %1-2
227 movu m %+ %%i, [ptrq + strideq*4]
228 add strideq, widthq
229 %assign %%i %%i+1
230 %endrep
231 movu m %+ %%i, [ptrq + strideq*4]
232%endmacro
233
234%macro SCATTERPS_INTERNAL 8-16
235 movu [ptrq + strideq*0], m %+ %1
236 mov strideq, widthq
237 %rotate 1
238 %rep %0-2
239 movu [ptrq + strideq*4], m %+ %1
240 add strideq, widthq
241 %rotate 1
242 %endrep
243 movu [ptrq + strideq*4], m %+ %1
244%endmacro
245
246%macro BATCH_INSERT64X4 4-*
247 %assign %%imm8 %1
248 %rotate 1
249 %rep (%0-1)/3
250 vinserti64x4 m%1, m%2, ym%3, %%imm8
251 %rotate 3
252 %endrep
253%endmacro
254
255%macro BATCH_EXTRACT_INSERT 2-*
256 %assign %%imm8 %1
257 %rotate 1
258 %rep (%0-1)/2
259 vextractf64x4 ym%1, m%1, %%imm8
260 vextractf64x4 ym%2, m%2, %%imm8
261 vinserti64x4 m%1, m%1, ym%2, %%imm8
262 %rotate 2
263 %endrep
264%endmacro
265
266%macro BATCH_MOVE 2-*
267 %rep %0/2
268 mova m%1, m%2
269 %rotate 2
270 %endrep
271%endmacro
272
273%macro BATCH_PERMUTE 3-*
274 %xdefine %%decorator %1
275 %xdefine %%mask %2
276 %assign %%index %3
277 %rotate 3
278 %rep (%0-3)/2
279 vperm %+ %%decorator m%1{%%mask}, m %+ %%index, m%2
280 %rotate 2
281 %endrep
282%endmacro
64 283
284; input : m3-m19
285; output: m8 m5 m9 m15 m16 m7 m17 m27 m24 m21 m25 m19 m12 m23 m13 m11
286%macro TRANSPOSE_16X16_AVX512 0
287 BATCH_INSERT64X4 0x1, 20,4,12, 21,5,13, 22,6,14, 23,7,15
288 BATCH_INSERT64X4 0x1, 24,8,16, 25,9,17, 26,10,18, 27,11,19
289
290 BATCH_EXTRACT_INSERT 0x1, 4,12, 5,13, 6,14, 7,15
291 BATCH_EXTRACT_INSERT 0x1, 8,16, 9,17, 10,18, 11,19
292
293 BATCH_MOVE 12,20, 13,21, 14,22, 15,23
294 BATCH_PERMUTE q, k6, 28, 12,24, 13,25, 14,26, 15,27
295 BATCH_PERMUTE q, k5, 28, 24,20, 25,21, 26,22, 27,23
296
297 BATCH_MOVE 16,4, 17,5, 18,6, 19,7
298 BATCH_PERMUTE q, k6, 28, 16,8, 17,9, 18,10, 19,11
299 BATCH_PERMUTE q, k5, 28, 8,4, 9,5, 10,6, 11,7
300
301 BATCH_MOVE 4,12, 5,13, 6,24, 7,25
302 BATCH_MOVE 20,16, 21,17, 22,8, 23,9
303
304 BATCH_PERMUTE q, k4, 29, 4,14, 5,15, 6,26, 7,27
305 BATCH_PERMUTE q, k3, 29, 14,12, 15,13, 26,24, 27,25
306 BATCH_PERMUTE q, k4, 29, 20,18, 21,19, 22,10, 23,11
307 BATCH_PERMUTE q, k3, 29, 18,16, 19,17, 10,8, 11,9
308
309 BATCH_MOVE 8,4, 9,14, 16,6, 17,26
310 BATCH_MOVE 24,20, 25,18, 12,22, 13,10
311
312 BATCH_PERMUTE d, k2, 30, 8,5, 9,15, 16,7, 17,27
313 BATCH_PERMUTE d, k1, 30, 5,4, 15,14, 7,6, 27,26
314 BATCH_PERMUTE d, k2, 30, 24,21, 25,19, 12,23, 13,11
315 BATCH_PERMUTE d, k1, 30, 21,20, 19,18, 23,22, 11,10
316%endmacro
317
318%macro INSERT_UNPACK 8
319 vinsertf128 m%5, m%1, xm%3, 0x1
320 vinsertf128 m%6, m%2, xm%4, 0x1
321 vunpcklpd m%7, m%5, m%6
322 vunpckhpd m%8, m%5, m%6
323%endmacro
324
325%macro SHUFFLE 4
326 vshufps m%3, m%1, m%2, 0x88
327 vshufps m%4, m%1, m%2, 0xDD
328 mova m%1, m%3
329 mova m%2, m%4
330%endmacro
331
332%macro EXTRACT_INSERT_UNPACK 6
333 vextractf128 xm%1, m%1, 0x1
334 vextractf128 xm%2, m%2, 0x1
335 vinsertf128 m%3, m%3, xm%1, 0x0
336 vinsertf128 m%4, m%4, xm%2, 0x0
337 vunpcklpd m%5, m%3, m%4
338 vunpckhpd m%6, m%3, m%4
339%endmacro
340
341; Transpose 8x8 AVX2
342; Limit the number ym# register to 16 for compatibility
343; Used up registers instead of using stack memory
344; Input: m2-m9
345; Output: m12, m14, m13, m15, m8, m10, m9, m11
346%macro TRANSPOSE_8X8_AVX2 0
347 INSERT_UNPACK 2, 3, 6, 7, 10, 11, 12, 13
348 INSERT_UNPACK 4, 5, 8, 9, 10, 11, 14, 15
349
350 SHUFFLE 12, 14, 10, 11
351 SHUFFLE 13, 15, 10, 11
352
353 EXTRACT_INSERT_UNPACK 4, 5, 8, 9, 10, 11
354 EXTRACT_INSERT_UNPACK 2, 3, 6, 7, 8, 9
355
356 SHUFFLE 8, 10, 6, 7
357 SHUFFLE 9, 11, 6, 7
358%endmacro
359
360%macro TRANSPOSE 0
361 %if cpuflag(avx512)
362 TRANSPOSE_16X16_AVX512
363 %elif cpuflag(avx2)
364 TRANSPOSE_8X8_AVX2
365 %endif
366%endmacro
367
368%macro WRITE_LOCAL_BUFFER 0
369 %if cpuflag(avx512)
370 WRITE_LOCAL_BUFFER_INTERNAL 8, 5, 9, 15, 16, 7, 17, 27, \
371 24, 21, 25, 19, 12, 23, 13, 11
372 %elif cpuflag(avx2)
373 WRITE_LOCAL_BUFFER_INTERNAL 12, 14, 13, 15, 8, 10, 9, 11
374 %endif
375%endmacro
376
377%macro SCATTERPS 0
378 %if cpuflag(avx512)
379 SCATTERPS_INTERNAL 8, 5, 9, 15, 16, 7, 17, 27, \
380 24, 21, 25, 19, 12, 23, 13, 11
381 %elif cpuflag(avx2)
382 SCATTERPS_INTERNAL 12, 14, 13, 15, 8, 10, 9, 11
383 %endif
384%endmacro
385
386%macro OPTIMIZED_LOOP_STEP 0
387 lea stepd, [stepsd - 1]
388 cmp stepd, 0
389 jle %%bscale_scalar
390%%loop_step:
391 sub localbufq, mmsize
392 mulps m3, m1
393 movu [localbufq], m3
394
395 ; Filter leftwards
396 lea xq, [widthq - 1]
397 %%loop_step_x_back:
398 sub localbufq, mmsize
399 FMULADD_PS m3, m3, m0, [localbufq], m3
400 movu [localbufq], m3
401
402 dec xq
403 cmp xq, 0
404 jg %%loop_step_x_back
405
406 ; Filter rightwards
407 mulps m3, m1
408 movu [localbufq], m3
409 add localbufq, mmsize
410
411 lea xq, [widthq - 1]
412 %%loop_step_x:
413 FMULADD_PS m3, m3, m0, [localbufq], m3
414 movu [localbufq], m3
415 add localbufq, mmsize
416
417 dec xq
418 cmp xq, 0
419 jg %%loop_step_x
420
421 dec stepd
422 cmp stepd, 0
423 jg %%loop_step
424
425%%bscale_scalar:
426%endmacro
427
428;***************************************************************************
429; void ff_horiz_slice(float *ptr, int width, int height, int steps,
430; float nu, float bscale)
431;***************************************************************************
65%macro HORIZ_SLICE 0432%macro HORIZ_SLICE 0
66%if UNIX64433%if UNIX64
434%if cpuflag(avx512) || cpuflag(avx2)
435cglobal horiz_slice, 5, 12, mmnum, 0-mmsize*4, buffer, width, height, steps, \
436 localbuf, x, y, step, stride, remain, ptr, mask
437%else
67cglobal horiz_slice, 4, 9, 9, ptr, width, height, steps, x, y, step, stride, remain438cglobal horiz_slice, 4, 9, 9, ptr, width, height, steps, x, y, step, stride, remain
439%endif
440%else
441%if cpuflag(avx512) || cpuflag(avx2)
442cglobal horiz_slice, 5, 12, mmnum, 0-mmsize*4, buffer, width, height, steps, nu, bscale, \
443 localbuf, x, y, step, stride, remain, ptr, mask
68%else444%else
69cglobal horiz_slice, 4, 9, 9, ptr, width, height, steps, nu, bscale, x, y, step, stride, remain445cglobal horiz_slice, 4, 9, 9, ptr, width, height, steps, nu, bscale, x, y, step, stride, remain
70%endif446%endif
447%endif
448%if cpuflag(avx512) || cpuflag(avx2)
449%assign rows mmsize/4
450%assign cols mmsize/4
451%if WIN64
452 VBROADCASTSS m0, num ; nu
453 VBROADCASTSS m1, bscalem ; bscale
454
455 mov nuq, localbufm
456 DEFINE_ARGS buffer, width, height, steps, \
457 localbuf, x, y, step, stride, remain, ptr, mask
458 MOVSXDIFNIDN width, height, steps
459%else
460 VBROADCASTSS m0, xmm0 ; nu
461 VBROADCASTSS m1, xmm1 ; bscale
462%endif
463
464%if cpuflag(avx512)
465 vpbroadcastd m2, widthd
466 INIT_WORD_MASK k6, k5, k4, k3, k2, k1
467 INIT_INDICES m28, m29, m30
468%else
469 movd xm2, widthd
470 VBROADCASTSS m2, xm2
471%endif
472
473 vpmulld m2, m2, [gblur_vindex_width] ; vindex width
474
475 xor yq, yq ; y = 0
476 xor xq, xq ; x = 0
477
478 cmp heightq, rows
479 jl .y_scalar
480 sub heightq, rows
481
482.loop_y:
483 ; ptr = buffer + y * width;
484 mov ptrq, yq
485 imul ptrq, widthq
486 lea ptrq, [bufferq + ptrq*4]
487
488 KXNOR m5, k7
489 VGATHERDPS m3, [ptrq + m2*4], m5, k7
490 mulps m3, m1
491 movu [localbufq], m3
492 add ptrq, 4
493 add localbufq, mmsize
494
495 ; Filter rightwards
496 PUSH_MM m2
497 lea xq, [widthq - 1]
498 .loop_x:
499 PUSH_MM m3
500 GATHERPS cols
501 TRANSPOSE
502 POP_MM m3
503 WRITE_LOCAL_BUFFER
504
505 add ptrq, mmsize
506 add localbufq, rows * mmsize
507 sub xq, cols
508 cmp xq, cols
509 jge .loop_x
510 POP_MM m2
511
512 cmp xq, 0
513 jle .bscale_scalar
514 .loop_x_scalar:
515 KXNOR m5, k7
516 VGATHERDPS m4, [ptrq + m2*4], m5, k7
517 FMULADD_PS m3, m3, m0, m4, m3
518 movu [localbufq], m3
519
520 add ptrq, 0x4
521 add localbufq, mmsize
522 dec xq
523 cmp xq, 0
524 jg .loop_x_scalar
525
526 OPTIMIZED_LOOP_STEP
527
528 .bscale_scalar:
529 sub ptrq, 4
530 sub localbufq, mmsize
531 mulps m3, m1
532 KXNOR m5, k7, maskq
533 VSCATTERDPS ptrq, strideq, m2, maskq, k7, 3, widthq, remainq
534
535 ; Filter leftwards
536 PUSH_MM m2
537 lea xq, [widthq - 1]
538 .loop_x_back:
539 sub localbufq, rows * mmsize
540 READ_LOCAL_BUFFER cols
541 PUSH_MM m2
542 TRANSPOSE
543 POP_MM m3
544 sub ptrq, mmsize
545 SCATTERPS
546
547 sub xq, cols
548 cmp xq, cols
549 jge .loop_x_back
550 POP_MM m2
551
552 cmp xq, 0
553 jle .end_loop_x
554 .loop_x_back_scalar:
555 sub ptrq, 0x4
556 sub localbufq, mmsize
557 FMULADD_PS m3, m3, m0, [localbufq], m3
558 KXNOR m5, k7, maskq
559 VSCATTERDPS ptrq, strideq, m2, maskq, k7, 3, widthq, remainq
560
561 dec xq
562 cmp xq, 0
563 jg .loop_x_back_scalar
564
565 .end_loop_x:
566
567 add yq, rows
568 cmp yq, heightq
569 jle .loop_y
570
571 add heightq, rows
572 cmp yq, heightq
573 jge .end_scalar
574
575 mov remainq, widthq
576 imul remainq, mmsize
577 add ptrq, remainq
578
579.y_scalar:
580 mov remainq, heightq
581 sub remainq, yq
582 mov maskq, 1
583 shlx maskq, maskq, remainq
584 sub maskq, 1
585 mov remainq, maskq
586 PUSH_MASK m5, k1, remaind, xd, rsp + 0x20
587
588 mov ptrq, yq
589 imul ptrq, widthq
590 lea ptrq, [bufferq + ptrq * 4] ; ptrq = buffer + y * width
591 KMOVW m6, m5, k7, k1
592 VGATHERDPS m3, [ptrq + m2 * 4], m6, k7
593 mulps m3, m1 ; p0 *= bscale
594 movu [localbufq], m3
595 add localbufq, mmsize
596
597 ; Filter rightwards
598 lea xq, [widthq - 1]
599 .y_scalar_loop_x:
600 add ptrq, 4
601 KMOVW m6, m5, k7, k1
602 VGATHERDPS m4, [ptrq + m2 * 4], m6, k7
603 FMULADD_PS m3, m3, m0, m4, m3
604 movu [localbufq], m3
605 add localbufq, mmsize
606
607 dec xq
608 cmp xq, 0
609 jg .y_scalar_loop_x
610
611 OPTIMIZED_LOOP_STEP
612
613 sub localbufq, mmsize
614 mulps m3, m1 ; p0 *= bscale
615 KMOVW k7, k1
616 VSCATTERDPS ptrq, strideq, m2, maskq, k7, 3, widthq, remainq, heightq
617
618 ; Filter leftwards
619 lea xq, [widthq - 1]
620 .y_scalar_loop_x_back:
621 sub ptrq, 4
622 sub localbufq, mmsize
623 FMULADD_PS m3, m3, m0, [localbufq], m3
624 KMOVW k7, k1
625 VSCATTERDPS ptrq, strideq, m2, maskq, k7, 3, widthq, remainq, heightq
626 dec xq
627 cmp xq, 0
628 jg .y_scalar_loop_x_back
629
630.end_scalar:
631 RET
632%else
71%if WIN64633%if WIN64
72 movss m0, num634 movss m0, num
73 movss m1, bscalem635 movss m1, bscalem
@@ -211,16 +773,26 @@ cglobal horiz_slice, 4, 9, 9, ptr, width, height, steps, nu, bscale, x, y, step, · 137 unmodified lines
211 jl .loop_y773 jl .loop_y
212 774
213 RET775 RET
776%endif
214%endmacro777%endmacro
215 778
216%if ARCH_X86_64779%if ARCH_X86_64
217INIT_XMM sse4780INIT_XMM sse4
218HORIZ_SLICE781HORIZ_SLICE
219 782
220INIT_XMM avx2783%if HAVE_AVX2_EXTERNAL
784INIT_YMM avx2
785%xdefine mmnum 16
221HORIZ_SLICE786HORIZ_SLICE
222%endif787%endif
223 788
789%if HAVE_AVX512_EXTERNAL
790INIT_ZMM avx512
791%xdefine mmnum 32
792HORIZ_SLICE
793%endif
794%endif
795
224%macro POSTSCALE_SLICE 0796%macro POSTSCALE_SLICE 0
225cglobal postscale_slice, 2, 2, 4, ptr, length, postscale, min, max797cglobal postscale_slice, 2, 2, 4, ptr, length, postscale, min, max
226 shl lengthd, 2798 shl lengthd, 2
@@ -270,7 +842,6 @@ INIT_ZMM avx512 · 43 unmodified lines
270POSTSCALE_SLICE842POSTSCALE_SLICE
271%endif843%endif
272 844
273
274;*******************************************************************************845;*******************************************************************************
275; void ff_verti_slice(float *buffer, int width, int height, int column_begin,846; void ff_verti_slice(float *buffer, int width, int height, int column_begin,
276; int column_end, int steps, float nu, float bscale);847; int column_end, int steps, float nu, float bscale);
libavfilter/x86/vf_gblur_init.c+14−3
@@ -24,8 +24,9 @@ · 23 unmodified lines
24#include "libavutil/x86/cpu.h"24#include "libavutil/x86/cpu.h"
25#include "libavfilter/gblur.h"25#include "libavfilter/gblur.h"
26 26
27void ff_horiz_slice_sse4(float *ptr, int width, int height, int steps, float nu, float bscale);27void ff_horiz_slice_sse4(float *ptr, int width, int height, int steps, float nu, float bscale, float *localbuf);
28void ff_horiz_slice_avx2(float *ptr, int width, int height, int steps, float nu, float bscale);28void ff_horiz_slice_avx2(float *ptr, int width, int height, int steps, float nu, float bscale, float *localbuf);
29void ff_horiz_slice_avx512(float *ptr, int width, int height, int steps, float nu, float bscale, float *localbuf);
29 30
30void ff_postscale_slice_sse(float *ptr, int length, float postscale, float min, float max);31void ff_postscale_slice_sse(float *ptr, int length, float postscale, float min, float max);
31void ff_postscale_slice_avx2(float *ptr, int length, float postscale, float min, float max);32void ff_postscale_slice_avx2(float *ptr, int length, float postscale, float min, float max);
@@ -51,12 +52,22 @@ av_cold void ff_gblur_init_x86(GBlurContext *s) · 19 unmodified lines
51 s->horiz_slice = ff_horiz_slice_sse4;52 s->horiz_slice = ff_horiz_slice_sse4;
52 }53 }
53 if (EXTERNAL_AVX2(cpu_flags)) {54 if (EXTERNAL_AVX2(cpu_flags)) {
54 s->horiz_slice = ff_horiz_slice_avx2;
55 s->verti_slice = ff_verti_slice_avx2;55 s->verti_slice = ff_verti_slice_avx2;
56 }56 }
57 if (EXTERNAL_AVX512(cpu_flags)) {57 if (EXTERNAL_AVX512(cpu_flags)) {
58 s->postscale_slice = ff_postscale_slice_avx512;58 s->postscale_slice = ff_postscale_slice_avx512;
59 s->verti_slice = ff_verti_slice_avx512;59 s->verti_slice = ff_verti_slice_avx512;
60 }60 }
61 if (EXTERNAL_AVX2(cpu_flags)) {
62 s->stride = EXTERNAL_AVX512(cpu_flags) ? 16 : 8;
63 s->localbuf = av_malloc(s->stride * sizeof(float) * s->planewidth[0] * s->planeheight[0]);
64 if (!s->localbuf)
65 return;
66
67 s->horiz_slice = ff_horiz_slice_avx2;
68 if (EXTERNAL_AVX512(cpu_flags)) {
69 s->horiz_slice = ff_horiz_slice_avx512;
70 }
71 }
61#endif72#endif
62}73}

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t