libavfilter/x86/vf_convolution: add sobel filter optimization and unit test with intel AVX512 VNNI

This commit enabled assembly code with intel AVX512 VNNI and added unit test for sobel filter sobel_c: 4537 sobel_avx512icl 2136

bwang30 committed 4 years ago

Commit 3ab11dc1 parent 2d25f339 files · +362 −78Browse files

Showing the changes to libavfilter (4 of 9 files). Show all

Unified · Split

4 files

libavfilter/convolution.h +74 −0
libavfilter/vf_convolution.c +13 −78
libavfilter/x86/vf_convolution.asm +147 −0
libavfilter/x86/vf_convolution_init.c +18 −0

libavfilter/convolution.h+74−0

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21	#ifndef AVFILTER_CONVOLUTION_H
22	22	#define AVFILTER_CONVOLUTION_H
23	23	#include "avfilter.h"
	24	+#include "libavutil/intreadwrite.h"
24	25
25	26	enum MatrixMode {
26	27	MATRIX_SQUARE,
@@ -61,4 +62,77 @@ typedef struct ConvolutionContext { · 34 unmodified lines
61	62	} ConvolutionContext;
62	63
63	64	void ff_convolution_init_x86(ConvolutionContext *s);
	65	+void ff_sobel_init_x86(ConvolutionContext *s, int depth, int nb_planes);
	66	+
	67	+static void setup_3x3(int radius, const uint8_t c[], const uint8_t src, int stride,
	68	+ int x, int w, int y, int h, int bpc)
	69	+{
	70	+ int i;
	71	+
	72	+ for (i = 0; i < 9; i++) {
	73	+ int xoff = FFABS(x + ((i % 3) - 1));
	74	+ int yoff = FFABS(y + (i / 3) - 1);
	75	+
	76	+ xoff = xoff >= w ? 2 * w - 1 - xoff : xoff;
	77	+ yoff = yoff >= h ? 2 * h - 1 - yoff : yoff;
	78	+
	79	+ c[i] = src + xoff * bpc + yoff * stride;
	80	+ }
	81	+}
	82	+
	83	+static void filter_sobel(uint8_t *dst, int width,
	84	+ float scale, float delta, const int *const matrix,
	85	+ const uint8_t *c[], int peak, int radius,
	86	+ int dstride, int stride, int size)
	87	+{
	88	+ const uint8_t c0 = c[0], c1 = c[1], *c2 = c[2];
	89	+ const uint8_t c3 = c[3], c5 = c[5];
	90	+ const uint8_t c6 = c[6], c7 = c[7], *c8 = c[8];
	91	+ int x;
	92	+
	93	+ for (x = 0; x < width; x++) {
	94	+ float suma = c0[x] * -1 + c1[x] * -2 + c2[x] * -1 +
	95	+ c6[x] * 1 + c7[x] * 2 + c8[x] * 1;
	96	+ float sumb = c0[x] * -1 + c2[x] * 1 + c3[x] * -2 +
	97	+ c5[x] * 2 + c6[x] * -1 + c8[x] * 1;
	98	+
	99	+ dst[x] = av_clip_uint8(sqrtf(sumasuma + sumbsumb) * scale + delta);
	100	+ }
	101	+}
	102	+
	103	+static void filter16_sobel(uint8_t *dstp, int width,
	104	+ float scale, float delta, const int *const matrix,
	105	+ const uint8_t *c[], int peak, int radius,
	106	+ int dstride, int stride, int size)
	107	+{
	108	+ uint16_t dst = (uint16_t )dstp;
	109	+ int x;
	110	+
	111	+ for (x = 0; x < width; x++) {
	112	+ float suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -2 + AV_RN16A(&c[2][2 * x]) * -1 +
	113	+ AV_RN16A(&c[6][2 * x]) * 1 + AV_RN16A(&c[7][2 * x]) * 2 + AV_RN16A(&c[8][2 * x]) * 1;
	114	+ float sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -2 +
	115	+ AV_RN16A(&c[5][2 * x]) * 2 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) * 1;
	116	+
	117	+ dst[x] = av_clip(sqrtf(sumasuma + sumbsumb) * scale + delta, 0, peak);
	118	+ }
	119	+}
	120	+
	121	+static av_unused void ff_sobel_init(ConvolutionContext *s, int depth, int nb_planes)
	122	+{
	123	+ for (int i = 0; i < 4; i++) {
	124	+ s->filter[i] = filter_sobel;
	125	+ s->copy[i] = !((1 << i) & s->planes);
	126	+ s->size[i] = 3;
	127	+ s->setup[i] = setup_3x3;
	128	+ s->rdiv[i] = s->scale;
	129	+ s->bias[i] = s->delta;
	130	+ }
	131	+ if (s->depth > 8)
	132	+ for (int i = 0; i < 4; i++)
	133	+ s->filter[i] = filter16_sobel;
	134	+#if ARCH_X86_64
	135	+ ff_sobel_init_x86(s, depth, nb_planes);
	136	+#endif
	137	+}
64	138	#endif

libavfilter/vf_convolution.c+13−78

@@ -139,24 +139,6 @@ static void filter16_roberts(uint8_t *dstp, int width, · 138 unmodified lines
139	139	}
140	140	}
141	141
142		−static void filter16_sobel(uint8_t *dstp, int width,
143		− float scale, float delta, const int *const matrix,
144		− const uint8_t *c[], int peak, int radius,
145		− int dstride, int stride, int size)
146		−{
147		− uint16_t dst = (uint16_t )dstp;
148		− int x;
149		−
150		− for (x = 0; x < width; x++) {
151		− float suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -2 + AV_RN16A(&c[2][2 * x]) * -1 +
152		− AV_RN16A(&c[6][2 * x]) * 1 + AV_RN16A(&c[7][2 * x]) * 2 + AV_RN16A(&c[8][2 * x]) * 1;
153		− float sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -2 +
154		− AV_RN16A(&c[5][2 * x]) * 2 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) * 1;
155		−
156		− dst[x] = av_clip(sqrtf(sumasuma + sumbsumb) * scale + delta, 0, peak);
157		− }
158		−}
159		−
160	142	static void filter16_scharr(uint8_t *dstp, int width,
161	143	float scale, float delta, const int *const matrix,
162	144	const uint8_t *c[], int peak, int radius,
@@ -261,26 +243,6 @@ static void filter_roberts(uint8_t *dst, int width, · 98 unmodified lines
261	243	}
262	244	}
263	245
264		−static void filter_sobel(uint8_t *dst, int width,
265		− float scale, float delta, const int *const matrix,
266		− const uint8_t *c[], int peak, int radius,
267		− int dstride, int stride, int size)
268		−{
269		− const uint8_t c0 = c[0], c1 = c[1], *c2 = c[2];
270		− const uint8_t c3 = c[3], c5 = c[5];
271		− const uint8_t c6 = c[6], c7 = c[7], *c8 = c[8];
272		− int x;
273		−
274		− for (x = 0; x < width; x++) {
275		− float suma = c0[x] * -1 + c1[x] * -2 + c2[x] * -1 +
276		− c6[x] * 1 + c7[x] * 2 + c8[x] * 1;
277		− float sumb = c0[x] * -1 + c2[x] * 1 + c3[x] * -2 +
278		− c5[x] * 2 + c6[x] * -1 + c8[x] * 1;
279		−
280		− dst[x] = av_clip_uint8(sqrtf(sumasuma + sumbsumb) * scale + delta);
281		− }
282		−}
283		−
284	246	static void filter_scharr(uint8_t *dst, int width,
285	247	float scale, float delta, const int *const matrix,
286	248	const uint8_t *c[], int peak, int radius,
@@ -552,22 +514,6 @@ static void filter_column(uint8_t *dst, int height, · 265 unmodified lines
552	514	}
553	515	}
554	516
555		−static void setup_3x3(int radius, const uint8_t c[], const uint8_t src, int stride,
556		− int x, int w, int y, int h, int bpc)
557		−{
558		− int i;
559		−
560		− for (i = 0; i < 9; i++) {
561		− int xoff = FFABS(x + ((i % 3) - 1));
562		− int yoff = FFABS(y + (i / 3) - 1);
563		−
564		− xoff = xoff >= w ? 2 * w - 1 - xoff : xoff;
565		− yoff = yoff >= h ? 2 * h - 1 - yoff : yoff;
566		−
567		− c[i] = src + xoff * bpc + yoff * stride;
568		− }
569		−}
570		−
571	517	static void setup_5x5(int radius, const uint8_t c[], const uint8_t src, int stride,
572	518	int x, int w, int y, int h, int bpc)
573	519	{
@@ -708,6 +654,18 @@ static int param_init(AVFilterContext *ctx) · 134 unmodified lines
708	654	const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
709	655	int p, i;
710	656
	657	+ s->depth = desc->comp[0].depth;
	658	+ s->max = (1 << s->depth) - 1;
	659	+
	660	+ s->planewidth[1] = s->planewidth[2] = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
	661	+ s->planewidth[0] = s->planewidth[3] = inlink->w;
	662	+ s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
	663	+ s->planeheight[0] = s->planeheight[3] = inlink->h;
	664	+
	665	+ s->nb_planes = av_pix_fmt_count_planes(inlink->format);
	666	+ s->nb_threads = ff_filter_get_nb_threads(ctx);
	667	+ s->bpc = (s->depth + 7) / 8;
	668	+
711	669	if (!strcmp(ctx->filter->name, "convolution")) {
712	670	for (i = 0; i < 4; i++) {
713	671	int matrix = (int )s->matrix[i];
@@ -804,14 +762,7 @@ static int param_init(AVFilterContext *ctx) · 90 unmodified lines
804	762	s->bias[i] = s->delta;
805	763	}
806	764	} else if (!strcmp(ctx->filter->name, "sobel")) {
807		− for (i = 0; i < 4; i++) {
808		− s->filter[i] = filter_sobel;
809		− s->copy[i] = !((1 << i) & s->planes);
810		− s->size[i] = 3;
811		− s->setup[i] = setup_3x3;
812		− s->rdiv[i] = s->scale;
813		− s->bias[i] = s->delta;
814		− }
	765	+ ff_sobel_init(s, s->depth, s->nb_planes);
815	766	} else if (!strcmp(ctx->filter->name, "kirsch")) {
816	767	for (i = 0; i < 4; i++) {
817	768	s->filter[i] = filter_kirsch;
@@ -832,18 +783,6 @@ static int param_init(AVFilterContext *ctx) · 14 unmodified lines
832	783	}
833	784	}
834	785
835		− s->depth = desc->comp[0].depth;
836		− s->max = (1 << s->depth) - 1;
837		−
838		− s->planewidth[1] = s->planewidth[2] = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
839		− s->planewidth[0] = s->planewidth[3] = inlink->w;
840		− s->planeheight[1] = s->planeheight[2] = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
841		− s->planeheight[0] = s->planeheight[3] = inlink->h;
842		−
843		− s->nb_planes = av_pix_fmt_count_planes(inlink->format);
844		− s->nb_threads = ff_filter_get_nb_threads(ctx);
845		− s->bpc = (s->depth + 7) / 8;
846		−
847	786	if (!strcmp(ctx->filter->name, "convolution")) {
848	787	if (s->depth > 8) {
849	788	for (p = 0; p < s->nb_planes; p++) {
@@ -870,10 +809,6 @@ static int param_init(AVFilterContext *ctx) · 20 unmodified lines
870	809	if (s->depth > 8)
871	810	for (p = 0; p < s->nb_planes; p++)
872	811	s->filter[p] = filter16_roberts;
873		− } else if (!strcmp(ctx->filter->name, "sobel")) {
874		− if (s->depth > 8)
875		− for (p = 0; p < s->nb_planes; p++)
876		− s->filter[p] = filter16_sobel;
877	812	} else if (!strcmp(ctx->filter->name, "kirsch")) {
878	813	if (s->depth > 8)
879	814	for (p = 0; p < s->nb_planes; p++)

libavfilter/x86/vf_convolution.asm+147−0

@@ -22,6 +22,18 @@ · 21 unmodified lines
22	22
23	23	SECTION_RODATA
24	24	half: dd 0.5
	25	+data_p1: dd 1
	26	+data_n1: dd -1
	27	+data_p2: dd 2
	28	+data_n2: dd -2
	29	+
	30	+ALIGN 64
	31	+sobel_perm: db 0, 16, 32, 48, 1, 17, 33, 49, 2, 18, 34, 50, 3, 19, 35, 51
	32	+ db 4, 20, 36, 52, 5, 21, 37, 53, 6, 22, 38, 54, 7, 23, 39, 55
	33	+ db 8, 24, 40, 56, 9, 25, 41, 57, 10, 26, 42, 58, 11, 27, 43, 59
	34	+ db 12, 28, 44, 60, 13, 29, 45, 61, 14, 30, 46, 62, 15, 31, 47, 63
	35	+sobel_mulA: db -1, 1, -2, 2
	36	+sobel_mulB: db 1, -1, 2, -2
25	37
26	38	SECTION .text
27	39
@@ -154,3 +166,138 @@ cglobal filter_3x3, 4, 15, 7, dst, width, rdiv, bias, matrix, ptr, c0, c1, c2, c · 126 unmodified lines
154	166	INIT_XMM sse4
155	167	FILTER_3X3
156	168	%endif
	169	+
	170	+%macro SOBEL_MUL 2
	171	+ movzx ptrd, byte [c%1q + xq]
	172	+ imul ptrd, [%2]
	173	+ add rd, ptrd
	174	+%endmacro
	175	+
	176	+%macro SOBEL_ADD 1
	177	+ movzx ptrd, byte [c%1q + xq]
	178	+ add rd, ptrd
	179	+%endmacro
	180	+
	181	+; void filter_sobel_avx512(uint8_t *dst, int width,
	182	+; float scale, float delta, const int *const matrix,
	183	+; const uint8_t *c[], int peak, int radius,
	184	+; int dstride, int stride)
	185	+%macro FILTER_SOBEL 0
	186	+%if UNIX64
	187	+cglobal filter_sobel, 4, 15, 7, dst, width, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x
	188	+%else
	189	+cglobal filter_sobel, 4, 15, 7, dst, width, rdiv, bias, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x
	190	+%endif
	191	+%if WIN64
	192	+ SWAP xmm0, xmm2
	193	+ SWAP xmm1, xmm3
	194	+ mov r2q, matrixmp
	195	+ mov r3q, ptrmp
	196	+ DEFINE_ARGS dst, width, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x
	197	+%endif
	198	+ movsxdifnidn widthq, widthd
	199	+ VBROADCASTSS m0, xmm0
	200	+ VBROADCASTSS m1, xmm1
	201	+ pxor m6, m6
	202	+ mov c0q, [ptrq + 0*gprsize]
	203	+ mov c1q, [ptrq + 1*gprsize]
	204	+ mov c2q, [ptrq + 2*gprsize]
	205	+ mov c3q, [ptrq + 3*gprsize]
	206	+ mov c4q, [ptrq + 4*gprsize]
	207	+ mov c5q, [ptrq + 5*gprsize]
	208	+ mov c6q, [ptrq + 6*gprsize]
	209	+ mov c7q, [ptrq + 7*gprsize]
	210	+ mov c8q, [ptrq + 8*gprsize]
	211	+
	212	+ xor xq, xq
	213	+ cmp widthq, mmsize/4
	214	+ jl .loop2
	215	+
	216	+ mov rq, widthq
	217	+ and rq, mmsize/4-1
	218	+ sub widthq, rq
	219	+
	220	+ mova m6, [sobel_perm]
	221	+.loop1:
	222	+ movu xm3, [c2q + xq]
	223	+ pmovzxbd m5, [c0q + xq]
	224	+ vinserti32x4 ym3, [c6q + xq], 1
	225	+ pmovzxbd m4, [c8q + xq]
	226	+ vinserti32x4 m2, m3, [c1q + xq], 2
	227	+ vinserti32x4 m3, [c5q + xq], 2
	228	+ vinserti32x4 m2, [c7q + xq], 3
	229	+ vinserti32x4 m3, [c3q + xq], 3
	230	+ vpermb m2, m6, m2
	231	+ psubd m4, m5
	232	+ vpermb m3, m6, m3
	233	+ mova m5, m4
	234	+ vpdpbusd m4, m2, [sobel_mulA] {1to16}
	235	+ vpdpbusd m5, m3, [sobel_mulB] {1to16}
	236	+
	237	+ cvtdq2ps m4, m4
	238	+ mulps m4, m4
	239	+
	240	+ cvtdq2ps m5, m5
	241	+ VFMADD231PS m4, m5, m5
	242	+
	243	+ sqrtps m4, m4
	244	+ fmaddps m4, m4, m0, m1
	245	+ cvttps2dq m4, m4
	246	+ vpmovusdb [dstq + xq], m4
	247	+
	248	+ add xq, mmsize/4
	249	+ cmp xq, widthq
	250	+ jl .loop1
	251	+
	252	+ add widthq, rq
	253	+ cmp xq, widthq
	254	+ jge .end
	255	+
	256	+.loop2:
	257	+ xor rd, rd
	258	+ pxor m4, m4
	259	+
	260	+ ;Gx
	261	+ SOBEL_MUL 0, data_n1
	262	+ SOBEL_MUL 1, data_n2
	263	+ SOBEL_MUL 2, data_n1
	264	+ SOBEL_ADD 6
	265	+ SOBEL_MUL 7, data_p2
	266	+ SOBEL_ADD 8
	267	+
	268	+ cvtsi2ss xmm4, rd
	269	+ mulss xmm4, xmm4
	270	+
	271	+ xor rd, rd
	272	+ ;Gy
	273	+ SOBEL_MUL 0, data_n1
	274	+ SOBEL_ADD 2
	275	+ SOBEL_MUL 3, data_n2
	276	+ SOBEL_MUL 5, data_p2
	277	+ SOBEL_MUL 6, data_n1
	278	+ SOBEL_ADD 8
	279	+
	280	+ cvtsi2ss xmm5, rd
	281	+ fmaddss xmm4, xmm5, xmm5, xmm4
	282	+
	283	+ sqrtps xmm4, xmm4
	284	+ fmaddss xmm4, xmm4, xmm0, xmm1 ;sum = sum * rdiv + bias
	285	+ cvttps2dq xmm4, xmm4 ; trunc to integer
	286	+ packssdw xmm4, xmm4
	287	+ packuswb xmm4, xmm4
	288	+ movd rd, xmm4
	289	+ mov [dstq + xq], rb
	290	+
	291	+ add xq, 1
	292	+ cmp xq, widthq
	293	+ jl .loop2
	294	+.end:
	295	+ RET
	296	+%endmacro
	297	+
	298	+%if ARCH_X86_64
	299	+%if HAVE_AVX512ICL_EXTERNAL
	300	+INIT_ZMM avx512icl
	301	+FILTER_SOBEL
	302	+%endif
	303	+%endif

libavfilter/x86/vf_convolution_init.c+18−0

@@ -29,6 +29,11 @@ void ff_filter_3x3_sse4(uint8_t *dst, int width, · 28 unmodified lines
29	29	const uint8_t *c[], int peak, int radius,
30	30	int dstride, int stride, int size);
31	31
	32	+void ff_filter_sobel_avx512icl(uint8_t *dst, int width,
	33	+ float scale, float delta, const int *const matrix,
	34	+ const uint8_t *c[], int peak, int radius,
	35	+ int dstride, int stride, int size);
	36	+
32	37	av_cold void ff_convolution_init_x86(ConvolutionContext *s)
33	38	{
34	39	#if ARCH_X86_64
@@ -44,3 +49,16 @@ av_cold void ff_convolution_init_x86(ConvolutionContext *s) · 9 unmodified lines
44	49	}
45	50	#endif
46	51	}
	52	+
	53	+av_cold void ff_sobel_init_x86(ConvolutionContext *s, int depth, int nb_planes)
	54	+{
	55	+#if ARCH_X86_64
	56	+ int cpu_flags = av_get_cpu_flags();
	57	+ for (int i = 0; i < nb_planes; i++) {
	58	+ if (depth == 8) {
	59	+ if (EXTERNAL_AVX512ICL(cpu_flags))
	60	+ s->filter[i] = ff_filter_sobel_avx512icl;
	61	+ }
	62	+ }
	63	+#endif
	64	+}

libavfilter/x86/vf_convolution: add sobel filter optimization and unit test with intel AVX512 VNNI

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code