History

swscale/x86: add sse4 and avx2 {lum,chr}ConvertRange16

chrRangeFromJpeg16_1920_c: 3153.9 chrRangeFromJpeg16_1920_sse4: 1770.0 (1.78x) chrRangeFromJpeg16_1920_avx2: 891.5 (3.54x) chrRangeToJpeg16_1920_c: 3165.0 chrRangeToJpeg16_1920_sse4: 1953.2 (1.62x) chrRangeToJpeg16_1920_avx2: 973.0 (3.25x) lumRangeFromJpeg16_1920_c: 1298.5 lumRangeFromJpeg16_1920_sse4: 886.5 (1.46x) lumRangeFromJpeg16_1920_avx2: 447.7 (2.90x) lumRangeToJpeg16_1920_c: 1905.0 lumRangeToJpeg16_1920_sse4: 993.0 (1.92x) lumRangeToJpeg16_1920_avx2: 498.9 (3.82x)

Ramiro Polla committed 2 years ago

Commit 87052c01 parent 6fe4a4f2 files · +122 −46Browse files

Unified · Split

2 files
  1. libswscale/x86/range_convert.asm +92 −22
  2. libswscale/x86/swscale.c +30 −24
libswscale/x86/range_convert.asm+92−22
@@ -20,21 +20,24 @@ · 19 unmodified lines
20 20
21%include "libavutil/x86/x86util.asm"21%include "libavutil/x86/x86util.asm"
22 22
23SECTION_RODATA
24pack19: times 4 dd (1 << 19) - 1
25
23SECTION .text26SECTION .text
24 27
25;-----------------------------------------------------------------------------28;-----------------------------------------------------------------------------
26; lumConvertRange29; lumConvertRange
27;30;
28; void ff_lumRangeToJpeg_<opt>(int16_t *dst, int width,31; void ff_lumRangeToJpeg{8,16}_<opt>(int16_t *dst, int width,
29; uint32_t coeff, int64_t offset);32; uint32_t coeff, int64_t offset);
30; void ff_lumRangeFromJpeg_<opt>(int16_t *dst, int width,33; void ff_lumRangeFromJpeg{8,16}_<opt>(int16_t *dst, int width,
31; uint32_t coeff, int64_t offset);34; uint32_t coeff, int64_t offset);
32;35;
33;-----------------------------------------------------------------------------36;-----------------------------------------------------------------------------
34 37
35%macro LUMCONVERTRANGE 138%macro LUMCONVERTRANGE 2
36cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset39cglobal lumRange%1Jpeg%2, 4, 4, 5, dst, width, coeff, offset
37 shl widthd, 140 shl widthd, %2 >> 3
38 movd xm2, coeffd41 movd xm2, coeffd
39 VBROADCASTSS m2, xm242 VBROADCASTSS m2, xm2
40%if ARCH_X86_6443%if ARCH_X86_64
@@ -42,12 +45,34 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 1 unmodified line
42%else45%else
43 movq xm3, offsetm46 movq xm3, offsetm
44%endif47%endif
48%if %2 == 16
49 VBROADCASTSD m3, xm3
50%ifidni %1,To
51 VBROADCASTI128 m4, [pack19]
52%endif
53%elif %2 == 8
45 VBROADCASTSS m3, xm354 VBROADCASTSS m3, xm3
46 pxor m4, m455 pxor m4, m4
56%endif ; %2 == 8/16
47 add dstq, widthq57 add dstq, widthq
48 neg widthq58 neg widthq
49.loop:59.loop:
50 movu m0, [dstq+widthq]60 movu m0, [dstq+widthq]
61%if %2 == 16
62 pshufd m1, m0, 0xb1
63 pmuldq m0, m2
64 pmuldq m1, m2
65 paddq m0, m3
66 paddq m1, m3
67 psrlq m0, 18
68 psrlq m1, 18
69 pshufd m0, m0, 0xd8
70 pshufd m1, m1, 0xd8
71 punpckldq m0, m1
72%ifidni %1,To
73 PMINSD m0, m4, m1
74%endif
75%elif %2 == 8
51 punpckhwd m1, m0, m476 punpckhwd m1, m0, m4
52 punpcklwd m0, m477 punpcklwd m0, m4
53 pmaddwd m0, m278 pmaddwd m0, m2
@@ -57,6 +82,7 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 3 unmodified lines
57 psrad m0, 1482 psrad m0, 14
58 psrad m1, 1483 psrad m1, 14
59 packssdw m0, m184 packssdw m0, m1
85%endif ; %2 == 8/16
60 movu [dstq+widthq], m086 movu [dstq+widthq], m0
61 add widthq, mmsize87 add widthq, mmsize
62 jl .loop88 jl .loop
@@ -66,16 +92,16 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 3 unmodified lines
66;-----------------------------------------------------------------------------92;-----------------------------------------------------------------------------
67; chrConvertRange93; chrConvertRange
68;94;
69; void ff_chrRangeToJpeg_<opt>(int16_t *dstU, int16_t *dstV, int width,95; void ff_chrRangeToJpeg{8,16}_<opt>(int16_t *dstU, int16_t *dstV, int width,
70; uint32_t coeff, int64_t offset);96; uint32_t coeff, int64_t offset);
71; void ff_chrRangeFromJpeg_<opt>(int16_t *dstU, int16_t *dstV, int width,97; void ff_chrRangeFromJpeg{8,16}_<opt>(int16_t *dstU, int16_t *dstV, int width,
72; uint32_t coeff, int64_t offset);98; uint32_t coeff, int64_t offset);
73;99;
74;-----------------------------------------------------------------------------100;-----------------------------------------------------------------------------
75 101
76%macro CHRCONVERTRANGE 1102%macro CHRCONVERTRANGE 2
77cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset103cglobal chrRange%1Jpeg%2, 5, 5, 7, dstU, dstV, width, coeff, offset
78 shl widthd, 1104 shl widthd, %2 >> 3
79 movd xm4, coeffd105 movd xm4, coeffd
80 VBROADCASTSS m4, xm4106 VBROADCASTSS m4, xm4
81%if ARCH_X86_64107%if ARCH_X86_64
@@ -83,14 +109,47 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 1 unmodified line
83%else109%else
84 movq xm5, offsetm110 movq xm5, offsetm
85%endif111%endif
112%if %2 == 16
113 VBROADCASTSD m5, xm5
114%ifidni %1,To
115 VBROADCASTI128 m6, [pack19]
116%endif
117%elif %2 == 8
86 VBROADCASTSS m5, xm5118 VBROADCASTSS m5, xm5
87 pxor m6, m6119 pxor m6, m6
120%endif ; %2 == 8/16
88 add dstUq, widthq121 add dstUq, widthq
89 add dstVq, widthq122 add dstVq, widthq
90 neg widthq123 neg widthq
91.loop:124.loop:
92 movu m0, [dstUq+widthq]125 movu m0, [dstUq+widthq]
93 movu m2, [dstVq+widthq]126 movu m2, [dstVq+widthq]
127%if %2 == 16
128 pshufd m1, m0, 0xb1
129 pshufd m3, m2, 0xb1
130 pmuldq m0, m4
131 pmuldq m1, m4
132 pmuldq m2, m4
133 pmuldq m3, m4
134 paddq m0, m5
135 paddq m1, m5
136 paddq m2, m5
137 paddq m3, m5
138 psrlq m0, 18
139 psrlq m1, 18
140 psrlq m2, 18
141 psrlq m3, 18
142 pshufd m0, m0, 0xd8
143 pshufd m1, m1, 0xd8
144 pshufd m2, m2, 0xd8
145 pshufd m3, m3, 0xd8
146 punpckldq m0, m1
147 punpckldq m2, m3
148%ifidni %1,To
149 PMINSD m0, m6, m1
150 PMINSD m2, m6, m3
151%endif
152%elif %2 == 8
94 punpckhwd m1, m0, m6153 punpckhwd m1, m0, m6
95 punpckhwd m3, m2, m6154 punpckhwd m3, m2, m6
96 punpcklwd m0, m6155 punpcklwd m0, m6
@@ -109,6 +168,7 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 12 unmodified lines
109 psrad m3, 14168 psrad m3, 14
110 packssdw m0, m1169 packssdw m0, m1
111 packssdw m2, m3170 packssdw m2, m3
171%endif ; %2 == 8/16
112 movu [dstUq+widthq], m0172 movu [dstUq+widthq], m0
113 movu [dstVq+widthq], m2173 movu [dstVq+widthq], m2
114 add widthq, mmsize174 add widthq, mmsize
@@ -117,15 +177,25 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 2 unmodified lines
117%endmacro177%endmacro
118 178
119INIT_XMM sse2179INIT_XMM sse2
120LUMCONVERTRANGE To180LUMCONVERTRANGE To, 8
121CHRCONVERTRANGE To181CHRCONVERTRANGE To, 8
122LUMCONVERTRANGE From182LUMCONVERTRANGE From, 8
123CHRCONVERTRANGE From183CHRCONVERTRANGE From, 8
184
185INIT_XMM sse4
186LUMCONVERTRANGE To, 16
187CHRCONVERTRANGE To, 16
188LUMCONVERTRANGE From, 16
189CHRCONVERTRANGE From, 16
124 190
125%if HAVE_AVX2_EXTERNAL191%if HAVE_AVX2_EXTERNAL
126INIT_YMM avx2192INIT_YMM avx2
127LUMCONVERTRANGE To193LUMCONVERTRANGE To, 8
128CHRCONVERTRANGE To194LUMCONVERTRANGE To, 16
129LUMCONVERTRANGE From195CHRCONVERTRANGE To, 8
130CHRCONVERTRANGE From196CHRCONVERTRANGE To, 16
197LUMCONVERTRANGE From, 8
198LUMCONVERTRANGE From, 16
199CHRCONVERTRANGE From, 8
200CHRCONVERTRANGE From, 16
131%endif201%endif
libswscale/x86/swscale.c+30−24
@@ -451,38 +451,44 @@ INPUT_PLANAR_RGB_UV_ALL_DECL(avx2); · 450 unmodified lines
451INPUT_PLANAR_RGB_A_ALL_DECL(avx2);451INPUT_PLANAR_RGB_A_ALL_DECL(avx2);
452#endif452#endif
453 453
454#define RANGE_CONVERT_FUNCS(opt) do { \454#define RANGE_CONVERT_FUNCS(opt, bpc) do { \
455 if (c->dstBpc <= 14) { \455 if (c->opts.src_range) { \
456 if (c->opts.src_range) { \456 c->lumConvertRange = ff_lumRangeFromJpeg##bpc##_##opt; \
457 c->lumConvertRange = ff_lumRangeFromJpeg_ ##opt; \457 c->chrConvertRange = ff_chrRangeFromJpeg##bpc##_##opt; \
458 c->chrConvertRange = ff_chrRangeFromJpeg_ ##opt; \458 } else { \
459 } else { \459 c->lumConvertRange = ff_lumRangeToJpeg##bpc##_##opt; \
460 c->lumConvertRange = ff_lumRangeToJpeg_ ##opt; \460 c->chrConvertRange = ff_chrRangeToJpeg##bpc##_##opt; \
461 c->chrConvertRange = ff_chrRangeToJpeg_ ##opt; \
462 } \
463 } \461 } \
464} while (0)462} while (0)
465 463
466#define RANGE_CONVERT_FUNCS_DECL(opt) \464#define RANGE_CONVERT_FUNCS_DECL(opt, bpc) \
467void ff_lumRangeFromJpeg_ ##opt(int16_t *dst, int width, \465void ff_lumRangeFromJpeg##bpc##_##opt(int16_t *dst, int width, \
468 uint32_t coeff, int64_t offset); \466 uint32_t coeff, int64_t offset); \
469void ff_chrRangeFromJpeg_ ##opt(int16_t *dstU, int16_t *dstV, int width, \467void ff_chrRangeFromJpeg##bpc##_##opt(int16_t *dstU, int16_t *dstV, int width, \
470 uint32_t coeff, int64_t offset); \468 uint32_t coeff, int64_t offset); \
471void ff_lumRangeToJpeg_ ##opt(int16_t *dst, int width, \469void ff_lumRangeToJpeg##bpc##_##opt(int16_t *dst, int width, \
472 uint32_t coeff, int64_t offset); \470 uint32_t coeff, int64_t offset); \
473void ff_chrRangeToJpeg_ ##opt(int16_t *dstU, int16_t *dstV, int width, \471void ff_chrRangeToJpeg##bpc##_##opt(int16_t *dstU, int16_t *dstV, int width, \
474 uint32_t coeff, int64_t offset); \472 uint32_t coeff, int64_t offset); \
475 473
476RANGE_CONVERT_FUNCS_DECL(sse2);474RANGE_CONVERT_FUNCS_DECL(sse2, 8)
477RANGE_CONVERT_FUNCS_DECL(avx2);475RANGE_CONVERT_FUNCS_DECL(sse4, 16)
476RANGE_CONVERT_FUNCS_DECL(avx2, 8)
477RANGE_CONVERT_FUNCS_DECL(avx2, 16)
478 478
479av_cold void ff_sws_init_range_convert_x86(SwsInternal *c)479av_cold void ff_sws_init_range_convert_x86(SwsInternal *c)
480{480{
481 int cpu_flags = av_get_cpu_flags();481 int cpu_flags = av_get_cpu_flags();
482 if (EXTERNAL_AVX2_FAST(cpu_flags)) {482 if (EXTERNAL_AVX2_FAST(cpu_flags)) {
483 RANGE_CONVERT_FUNCS(avx2);483 if (c->dstBpc <= 14) {
484 } else if (EXTERNAL_SSE2(cpu_flags)) {484 RANGE_CONVERT_FUNCS(avx2, 8);
485 RANGE_CONVERT_FUNCS(sse2);485 } else {
486 RANGE_CONVERT_FUNCS(avx2, 16);
487 }
488 } else if (EXTERNAL_SSE2(cpu_flags) && c->dstBpc <= 14) {
489 RANGE_CONVERT_FUNCS(sse2, 8);
490 } else if (EXTERNAL_SSE4(cpu_flags) && c->dstBpc > 14) {
491 RANGE_CONVERT_FUNCS(sse4, 16);
486 }492 }
487}493}
488 494

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t