History

swscale/x86: add sse4 and avx2 {lum,chr}ConvertRange16

chrRangeFromJpeg16_1920_c: 3153.9 chrRangeFromJpeg16_1920_sse4: 1770.0 (1.78x) chrRangeFromJpeg16_1920_avx2: 891.5 (3.54x) chrRangeToJpeg16_1920_c: 3165.0 chrRangeToJpeg16_1920_sse4: 1953.2 (1.62x) chrRangeToJpeg16_1920_avx2: 973.0 (3.25x) lumRangeFromJpeg16_1920_c: 1298.5 lumRangeFromJpeg16_1920_sse4: 886.5 (1.46x) lumRangeFromJpeg16_1920_avx2: 447.7 (2.90x) lumRangeToJpeg16_1920_c: 1905.0 lumRangeToJpeg16_1920_sse4: 993.0 (1.92x) lumRangeToJpeg16_1920_avx2: 498.9 (3.82x)

Ramiro Polla committed 2 years ago

Commit 87052c01 parent 6fe4a4f2 files · +122 −46Browse files

Unified · Split

2 files
  1. libswscale/x86/range_convert.asm +92 −22
  2. libswscale/x86/swscale.c +30 −24
libswscale/x86/range_convert.asm+92−22
@@ -20,21 +20,24 @@ · 19 unmodified lines
2020
2121 %include "libavutil/x86/x86util.asm"
2222
23+SECTION_RODATA
24+pack19: times 4 dd (1 << 19) - 1
25+
2326 SECTION .text
2427
2528 ;-----------------------------------------------------------------------------
2629 ; lumConvertRange
2730 ;
28−; void ff_lumRangeToJpeg_<opt>(int16_t *dst, int width,
29−; uint32_t coeff, int64_t offset);
30−; void ff_lumRangeFromJpeg_<opt>(int16_t *dst, int width,
31−; uint32_t coeff, int64_t offset);
31+; void ff_lumRangeToJpeg{8,16}_<opt>(int16_t *dst, int width,
32+; uint32_t coeff, int64_t offset);
33+; void ff_lumRangeFromJpeg{8,16}_<opt>(int16_t *dst, int width,
34+; uint32_t coeff, int64_t offset);
3235 ;
3336 ;-----------------------------------------------------------------------------
3437
35−%macro LUMCONVERTRANGE 1
36−cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset
37− shl widthd, 1
38+%macro LUMCONVERTRANGE 2
39+cglobal lumRange%1Jpeg%2, 4, 4, 5, dst, width, coeff, offset
40+ shl widthd, %2 >> 3
3841 movd xm2, coeffd
3942 VBROADCASTSS m2, xm2
4043 %if ARCH_X86_64
@@ -42,12 +45,34 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 1 unmodified line
4245 %else
4346 movq xm3, offsetm
4447 %endif
48+%if %2 == 16
49+ VBROADCASTSD m3, xm3
50+%ifidni %1,To
51+ VBROADCASTI128 m4, [pack19]
52+%endif
53+%elif %2 == 8
4554 VBROADCASTSS m3, xm3
4655 pxor m4, m4
56+%endif ; %2 == 8/16
4757 add dstq, widthq
4858 neg widthq
4959 .loop:
5060 movu m0, [dstq+widthq]
61+%if %2 == 16
62+ pshufd m1, m0, 0xb1
63+ pmuldq m0, m2
64+ pmuldq m1, m2
65+ paddq m0, m3
66+ paddq m1, m3
67+ psrlq m0, 18
68+ psrlq m1, 18
69+ pshufd m0, m0, 0xd8
70+ pshufd m1, m1, 0xd8
71+ punpckldq m0, m1
72+%ifidni %1,To
73+ PMINSD m0, m4, m1
74+%endif
75+%elif %2 == 8
5176 punpckhwd m1, m0, m4
5277 punpcklwd m0, m4
5378 pmaddwd m0, m2
@@ -57,6 +82,7 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 3 unmodified lines
5782 psrad m0, 14
5883 psrad m1, 14
5984 packssdw m0, m1
85+%endif ; %2 == 8/16
6086 movu [dstq+widthq], m0
6187 add widthq, mmsize
6288 jl .loop
@@ -66,16 +92,16 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 3 unmodified lines
6692 ;-----------------------------------------------------------------------------
6793 ; chrConvertRange
6894 ;
69−; void ff_chrRangeToJpeg_<opt>(int16_t *dstU, int16_t *dstV, int width,
70−; uint32_t coeff, int64_t offset);
71−; void ff_chrRangeFromJpeg_<opt>(int16_t *dstU, int16_t *dstV, int width,
72−; uint32_t coeff, int64_t offset);
95+; void ff_chrRangeToJpeg{8,16}_<opt>(int16_t *dstU, int16_t *dstV, int width,
96+; uint32_t coeff, int64_t offset);
97+; void ff_chrRangeFromJpeg{8,16}_<opt>(int16_t *dstU, int16_t *dstV, int width,
98+; uint32_t coeff, int64_t offset);
7399 ;
74100 ;-----------------------------------------------------------------------------
75101
76−%macro CHRCONVERTRANGE 1
77−cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset
78− shl widthd, 1
102+%macro CHRCONVERTRANGE 2
103+cglobal chrRange%1Jpeg%2, 5, 5, 7, dstU, dstV, width, coeff, offset
104+ shl widthd, %2 >> 3
79105 movd xm4, coeffd
80106 VBROADCASTSS m4, xm4
81107 %if ARCH_X86_64
@@ -83,14 +109,47 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 1 unmodified line
83109 %else
84110 movq xm5, offsetm
85111 %endif
112+%if %2 == 16
113+ VBROADCASTSD m5, xm5
114+%ifidni %1,To
115+ VBROADCASTI128 m6, [pack19]
116+%endif
117+%elif %2 == 8
86118 VBROADCASTSS m5, xm5
87119 pxor m6, m6
120+%endif ; %2 == 8/16
88121 add dstUq, widthq
89122 add dstVq, widthq
90123 neg widthq
91124 .loop:
92125 movu m0, [dstUq+widthq]
93126 movu m2, [dstVq+widthq]
127+%if %2 == 16
128+ pshufd m1, m0, 0xb1
129+ pshufd m3, m2, 0xb1
130+ pmuldq m0, m4
131+ pmuldq m1, m4
132+ pmuldq m2, m4
133+ pmuldq m3, m4
134+ paddq m0, m5
135+ paddq m1, m5
136+ paddq m2, m5
137+ paddq m3, m5
138+ psrlq m0, 18
139+ psrlq m1, 18
140+ psrlq m2, 18
141+ psrlq m3, 18
142+ pshufd m0, m0, 0xd8
143+ pshufd m1, m1, 0xd8
144+ pshufd m2, m2, 0xd8
145+ pshufd m3, m3, 0xd8
146+ punpckldq m0, m1
147+ punpckldq m2, m3
148+%ifidni %1,To
149+ PMINSD m0, m6, m1
150+ PMINSD m2, m6, m3
151+%endif
152+%elif %2 == 8
94153 punpckhwd m1, m0, m6
95154 punpckhwd m3, m2, m6
96155 punpcklwd m0, m6
@@ -109,6 +168,7 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 12 unmodified lines
109168 psrad m3, 14
110169 packssdw m0, m1
111170 packssdw m2, m3
171+%endif ; %2 == 8/16
112172 movu [dstUq+widthq], m0
113173 movu [dstVq+widthq], m2
114174 add widthq, mmsize
@@ -117,15 +177,25 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 2 unmodified lines
117177 %endmacro
118178
119179 INIT_XMM sse2
120−LUMCONVERTRANGE To
121−CHRCONVERTRANGE To
122−LUMCONVERTRANGE From
123−CHRCONVERTRANGE From
180+LUMCONVERTRANGE To, 8
181+CHRCONVERTRANGE To, 8
182+LUMCONVERTRANGE From, 8
183+CHRCONVERTRANGE From, 8
184+
185+INIT_XMM sse4
186+LUMCONVERTRANGE To, 16
187+CHRCONVERTRANGE To, 16
188+LUMCONVERTRANGE From, 16
189+CHRCONVERTRANGE From, 16
124190
125191 %if HAVE_AVX2_EXTERNAL
126192 INIT_YMM avx2
127−LUMCONVERTRANGE To
128−CHRCONVERTRANGE To
129−LUMCONVERTRANGE From
130−CHRCONVERTRANGE From
193+LUMCONVERTRANGE To, 8
194+LUMCONVERTRANGE To, 16
195+CHRCONVERTRANGE To, 8
196+CHRCONVERTRANGE To, 16
197+LUMCONVERTRANGE From, 8
198+LUMCONVERTRANGE From, 16
199+CHRCONVERTRANGE From, 8
200+CHRCONVERTRANGE From, 16
131201 %endif
libswscale/x86/swscale.c+30−24
@@ -451,38 +451,44 @@ INPUT_PLANAR_RGB_UV_ALL_DECL(avx2); · 450 unmodified lines
451451 INPUT_PLANAR_RGB_A_ALL_DECL(avx2);
452452 #endif
453453
454−#define RANGE_CONVERT_FUNCS(opt) do { \
455− if (c->dstBpc <= 14) { \
456− if (c->opts.src_range) { \
457− c->lumConvertRange = ff_lumRangeFromJpeg_ ##opt; \
458− c->chrConvertRange = ff_chrRangeFromJpeg_ ##opt; \
459− } else { \
460− c->lumConvertRange = ff_lumRangeToJpeg_ ##opt; \
461− c->chrConvertRange = ff_chrRangeToJpeg_ ##opt; \
462− } \
454+#define RANGE_CONVERT_FUNCS(opt, bpc) do { \
455+ if (c->opts.src_range) { \
456+ c->lumConvertRange = ff_lumRangeFromJpeg##bpc##_##opt; \
457+ c->chrConvertRange = ff_chrRangeFromJpeg##bpc##_##opt; \
458+ } else { \
459+ c->lumConvertRange = ff_lumRangeToJpeg##bpc##_##opt; \
460+ c->chrConvertRange = ff_chrRangeToJpeg##bpc##_##opt; \
463461 } \
464462 } while (0)
465463
466−#define RANGE_CONVERT_FUNCS_DECL(opt) \
467−void ff_lumRangeFromJpeg_ ##opt(int16_t *dst, int width, \
468− uint32_t coeff, int64_t offset); \
469−void ff_chrRangeFromJpeg_ ##opt(int16_t *dstU, int16_t *dstV, int width, \
470− uint32_t coeff, int64_t offset); \
471−void ff_lumRangeToJpeg_ ##opt(int16_t *dst, int width, \
472− uint32_t coeff, int64_t offset); \
473−void ff_chrRangeToJpeg_ ##opt(int16_t *dstU, int16_t *dstV, int width, \
474− uint32_t coeff, int64_t offset); \
475−
476−RANGE_CONVERT_FUNCS_DECL(sse2);
477−RANGE_CONVERT_FUNCS_DECL(avx2);
464+#define RANGE_CONVERT_FUNCS_DECL(opt, bpc) \
465+void ff_lumRangeFromJpeg##bpc##_##opt(int16_t *dst, int width, \
466+ uint32_t coeff, int64_t offset); \
467+void ff_chrRangeFromJpeg##bpc##_##opt(int16_t *dstU, int16_t *dstV, int width, \
468+ uint32_t coeff, int64_t offset); \
469+void ff_lumRangeToJpeg##bpc##_##opt(int16_t *dst, int width, \
470+ uint32_t coeff, int64_t offset); \
471+void ff_chrRangeToJpeg##bpc##_##opt(int16_t *dstU, int16_t *dstV, int width, \
472+ uint32_t coeff, int64_t offset); \
473+
474+RANGE_CONVERT_FUNCS_DECL(sse2, 8)
475+RANGE_CONVERT_FUNCS_DECL(sse4, 16)
476+RANGE_CONVERT_FUNCS_DECL(avx2, 8)
477+RANGE_CONVERT_FUNCS_DECL(avx2, 16)
478478
479479 av_cold void ff_sws_init_range_convert_x86(SwsInternal *c)
480480 {
481481 int cpu_flags = av_get_cpu_flags();
482482 if (EXTERNAL_AVX2_FAST(cpu_flags)) {
483− RANGE_CONVERT_FUNCS(avx2);
484− } else if (EXTERNAL_SSE2(cpu_flags)) {
485− RANGE_CONVERT_FUNCS(sse2);
483+ if (c->dstBpc <= 14) {
484+ RANGE_CONVERT_FUNCS(avx2, 8);
485+ } else {
486+ RANGE_CONVERT_FUNCS(avx2, 16);
487+ }
488+ } else if (EXTERNAL_SSE2(cpu_flags) && c->dstBpc <= 14) {
489+ RANGE_CONVERT_FUNCS(sse2, 8);
490+ } else if (EXTERNAL_SSE4(cpu_flags) && c->dstBpc > 14) {
491+ RANGE_CONVERT_FUNCS(sse4, 16);
486492 }
487493 }
488494

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t