chrRangeFromJpeg16_1920_c: 3153.9 chrRangeFromJpeg16_1920_sse4: 1770.0 (1.78x) chrRangeFromJpeg16_1920_avx2: 891.5 (3.54x) chrRangeToJpeg16_1920_c: 3165.0 chrRangeToJpeg16_1920_sse4: 1953.2 (1.62x) chrRangeToJpeg16_1920_avx2: 973.0 (3.25x) lumRangeFromJpeg16_1920_c: 1298.5 lumRangeFromJpeg16_1920_sse4: 886.5 (1.46x) lumRangeFromJpeg16_1920_avx2: 447.7 (2.90x) lumRangeToJpeg16_1920_c: 1905.0 lumRangeToJpeg16_1920_sse4: 993.0 (1.92x) lumRangeToJpeg16_1920_avx2: 498.9 (3.82x)
Unified · Split
| @@ -20,21 +20,24 @@ · 19 unmodified lines | ||
| 20 | 20 | |
| 21 | 21 | %include "libavutil/x86/x86util.asm" |
| 22 | 22 | |
| 23 | +SECTION_RODATA | |
| 24 | +pack19: times 4 dd (1 << 19) - 1 | |
| 25 | + | |
| 23 | 26 | SECTION .text |
| 24 | 27 | |
| 25 | 28 | ;----------------------------------------------------------------------------- |
| 26 | 29 | ; lumConvertRange |
| 27 | 30 | ; |
| 28 | −; void ff_lumRangeToJpeg_<opt>(int16_t *dst, int width, | |
| 29 | −; uint32_t coeff, int64_t offset); | |
| 30 | −; void ff_lumRangeFromJpeg_<opt>(int16_t *dst, int width, | |
| 31 | −; uint32_t coeff, int64_t offset); | |
| 31 | +; void ff_lumRangeToJpeg{8,16}_<opt>(int16_t *dst, int width, | |
| 32 | +; uint32_t coeff, int64_t offset); | |
| 33 | +; void ff_lumRangeFromJpeg{8,16}_<opt>(int16_t *dst, int width, | |
| 34 | +; uint32_t coeff, int64_t offset); | |
| 32 | 35 | ; |
| 33 | 36 | ;----------------------------------------------------------------------------- |
| 34 | 37 | |
| 35 | −%macro LUMCONVERTRANGE 1 | |
| 36 | −cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset | |
| 37 | − shl widthd, 1 | |
| 38 | +%macro LUMCONVERTRANGE 2 | |
| 39 | +cglobal lumRange%1Jpeg%2, 4, 4, 5, dst, width, coeff, offset | |
| 40 | + shl widthd, %2 >> 3 | |
| 38 | 41 | movd xm2, coeffd |
| 39 | 42 | VBROADCASTSS m2, xm2 |
| 40 | 43 | %if ARCH_X86_64 |
| @@ -42,12 +45,34 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 1 unmodified line | ||
| 42 | 45 | %else |
| 43 | 46 | movq xm3, offsetm |
| 44 | 47 | %endif |
| 48 | +%if %2 == 16 | |
| 49 | + VBROADCASTSD m3, xm3 | |
| 50 | +%ifidni %1,To | |
| 51 | + VBROADCASTI128 m4, [pack19] | |
| 52 | +%endif | |
| 53 | +%elif %2 == 8 | |
| 45 | 54 | VBROADCASTSS m3, xm3 |
| 46 | 55 | pxor m4, m4 |
| 56 | +%endif ; %2 == 8/16 | |
| 47 | 57 | add dstq, widthq |
| 48 | 58 | neg widthq |
| 49 | 59 | .loop: |
| 50 | 60 | movu m0, [dstq+widthq] |
| 61 | +%if %2 == 16 | |
| 62 | + pshufd m1, m0, 0xb1 | |
| 63 | + pmuldq m0, m2 | |
| 64 | + pmuldq m1, m2 | |
| 65 | + paddq m0, m3 | |
| 66 | + paddq m1, m3 | |
| 67 | + psrlq m0, 18 | |
| 68 | + psrlq m1, 18 | |
| 69 | + pshufd m0, m0, 0xd8 | |
| 70 | + pshufd m1, m1, 0xd8 | |
| 71 | + punpckldq m0, m1 | |
| 72 | +%ifidni %1,To | |
| 73 | + PMINSD m0, m4, m1 | |
| 74 | +%endif | |
| 75 | +%elif %2 == 8 | |
| 51 | 76 | punpckhwd m1, m0, m4 |
| 52 | 77 | punpcklwd m0, m4 |
| 53 | 78 | pmaddwd m0, m2 |
| @@ -57,6 +82,7 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 3 unmodified lines | ||
| 57 | 82 | psrad m0, 14 |
| 58 | 83 | psrad m1, 14 |
| 59 | 84 | packssdw m0, m1 |
| 85 | +%endif ; %2 == 8/16 | |
| 60 | 86 | movu [dstq+widthq], m0 |
| 61 | 87 | add widthq, mmsize |
| 62 | 88 | jl .loop |
| @@ -66,16 +92,16 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 3 unmodified lines | ||
| 66 | 92 | ;----------------------------------------------------------------------------- |
| 67 | 93 | ; chrConvertRange |
| 68 | 94 | ; |
| 69 | −; void ff_chrRangeToJpeg_<opt>(int16_t *dstU, int16_t *dstV, int width, | |
| 70 | −; uint32_t coeff, int64_t offset); | |
| 71 | −; void ff_chrRangeFromJpeg_<opt>(int16_t *dstU, int16_t *dstV, int width, | |
| 72 | −; uint32_t coeff, int64_t offset); | |
| 95 | +; void ff_chrRangeToJpeg{8,16}_<opt>(int16_t *dstU, int16_t *dstV, int width, | |
| 96 | +; uint32_t coeff, int64_t offset); | |
| 97 | +; void ff_chrRangeFromJpeg{8,16}_<opt>(int16_t *dstU, int16_t *dstV, int width, | |
| 98 | +; uint32_t coeff, int64_t offset); | |
| 73 | 99 | ; |
| 74 | 100 | ;----------------------------------------------------------------------------- |
| 75 | 101 | |
| 76 | −%macro CHRCONVERTRANGE 1 | |
| 77 | −cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset | |
| 78 | − shl widthd, 1 | |
| 102 | +%macro CHRCONVERTRANGE 2 | |
| 103 | +cglobal chrRange%1Jpeg%2, 5, 5, 7, dstU, dstV, width, coeff, offset | |
| 104 | + shl widthd, %2 >> 3 | |
| 79 | 105 | movd xm4, coeffd |
| 80 | 106 | VBROADCASTSS m4, xm4 |
| 81 | 107 | %if ARCH_X86_64 |
| @@ -83,14 +109,47 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 1 unmodified line | ||
| 83 | 109 | %else |
| 84 | 110 | movq xm5, offsetm |
| 85 | 111 | %endif |
| 112 | +%if %2 == 16 | |
| 113 | + VBROADCASTSD m5, xm5 | |
| 114 | +%ifidni %1,To | |
| 115 | + VBROADCASTI128 m6, [pack19] | |
| 116 | +%endif | |
| 117 | +%elif %2 == 8 | |
| 86 | 118 | VBROADCASTSS m5, xm5 |
| 87 | 119 | pxor m6, m6 |
| 120 | +%endif ; %2 == 8/16 | |
| 88 | 121 | add dstUq, widthq |
| 89 | 122 | add dstVq, widthq |
| 90 | 123 | neg widthq |
| 91 | 124 | .loop: |
| 92 | 125 | movu m0, [dstUq+widthq] |
| 93 | 126 | movu m2, [dstVq+widthq] |
| 127 | +%if %2 == 16 | |
| 128 | + pshufd m1, m0, 0xb1 | |
| 129 | + pshufd m3, m2, 0xb1 | |
| 130 | + pmuldq m0, m4 | |
| 131 | + pmuldq m1, m4 | |
| 132 | + pmuldq m2, m4 | |
| 133 | + pmuldq m3, m4 | |
| 134 | + paddq m0, m5 | |
| 135 | + paddq m1, m5 | |
| 136 | + paddq m2, m5 | |
| 137 | + paddq m3, m5 | |
| 138 | + psrlq m0, 18 | |
| 139 | + psrlq m1, 18 | |
| 140 | + psrlq m2, 18 | |
| 141 | + psrlq m3, 18 | |
| 142 | + pshufd m0, m0, 0xd8 | |
| 143 | + pshufd m1, m1, 0xd8 | |
| 144 | + pshufd m2, m2, 0xd8 | |
| 145 | + pshufd m3, m3, 0xd8 | |
| 146 | + punpckldq m0, m1 | |
| 147 | + punpckldq m2, m3 | |
| 148 | +%ifidni %1,To | |
| 149 | + PMINSD m0, m6, m1 | |
| 150 | + PMINSD m2, m6, m3 | |
| 151 | +%endif | |
| 152 | +%elif %2 == 8 | |
| 94 | 153 | punpckhwd m1, m0, m6 |
| 95 | 154 | punpckhwd m3, m2, m6 |
| 96 | 155 | punpcklwd m0, m6 |
| @@ -109,6 +168,7 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 12 unmodified lines | ||
| 109 | 168 | psrad m3, 14 |
| 110 | 169 | packssdw m0, m1 |
| 111 | 170 | packssdw m2, m3 |
| 171 | +%endif ; %2 == 8/16 | |
| 112 | 172 | movu [dstUq+widthq], m0 |
| 113 | 173 | movu [dstVq+widthq], m2 |
| 114 | 174 | add widthq, mmsize |
| @@ -117,15 +177,25 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 2 unmodified lines | ||
| 117 | 177 | %endmacro |
| 118 | 178 | |
| 119 | 179 | INIT_XMM sse2 |
| 120 | −LUMCONVERTRANGE To | |
| 121 | −CHRCONVERTRANGE To | |
| 122 | −LUMCONVERTRANGE From | |
| 123 | −CHRCONVERTRANGE From | |
| 180 | +LUMCONVERTRANGE To, 8 | |
| 181 | +CHRCONVERTRANGE To, 8 | |
| 182 | +LUMCONVERTRANGE From, 8 | |
| 183 | +CHRCONVERTRANGE From, 8 | |
| 184 | + | |
| 185 | +INIT_XMM sse4 | |
| 186 | +LUMCONVERTRANGE To, 16 | |
| 187 | +CHRCONVERTRANGE To, 16 | |
| 188 | +LUMCONVERTRANGE From, 16 | |
| 189 | +CHRCONVERTRANGE From, 16 | |
| 124 | 190 | |
| 125 | 191 | %if HAVE_AVX2_EXTERNAL |
| 126 | 192 | INIT_YMM avx2 |
| 127 | −LUMCONVERTRANGE To | |
| 128 | −CHRCONVERTRANGE To | |
| 129 | −LUMCONVERTRANGE From | |
| 130 | −CHRCONVERTRANGE From | |
| 193 | +LUMCONVERTRANGE To, 8 | |
| 194 | +LUMCONVERTRANGE To, 16 | |
| 195 | +CHRCONVERTRANGE To, 8 | |
| 196 | +CHRCONVERTRANGE To, 16 | |
| 197 | +LUMCONVERTRANGE From, 8 | |
| 198 | +LUMCONVERTRANGE From, 16 | |
| 199 | +CHRCONVERTRANGE From, 8 | |
| 200 | +CHRCONVERTRANGE From, 16 | |
| 131 | 201 | %endif |
| @@ -451,38 +451,44 @@ INPUT_PLANAR_RGB_UV_ALL_DECL(avx2); · 450 unmodified lines | ||
| 451 | 451 | INPUT_PLANAR_RGB_A_ALL_DECL(avx2); |
| 452 | 452 | #endif |
| 453 | 453 | |
| 454 | −#define RANGE_CONVERT_FUNCS(opt) do { \ | |
| 455 | − if (c->dstBpc <= 14) { \ | |
| 456 | − if (c->opts.src_range) { \ | |
| 457 | − c->lumConvertRange = ff_lumRangeFromJpeg_ ##opt; \ | |
| 458 | − c->chrConvertRange = ff_chrRangeFromJpeg_ ##opt; \ | |
| 459 | − } else { \ | |
| 460 | − c->lumConvertRange = ff_lumRangeToJpeg_ ##opt; \ | |
| 461 | − c->chrConvertRange = ff_chrRangeToJpeg_ ##opt; \ | |
| 462 | − } \ | |
| 454 | +#define RANGE_CONVERT_FUNCS(opt, bpc) do { \ | |
| 455 | + if (c->opts.src_range) { \ | |
| 456 | + c->lumConvertRange = ff_lumRangeFromJpeg##bpc##_##opt; \ | |
| 457 | + c->chrConvertRange = ff_chrRangeFromJpeg##bpc##_##opt; \ | |
| 458 | + } else { \ | |
| 459 | + c->lumConvertRange = ff_lumRangeToJpeg##bpc##_##opt; \ | |
| 460 | + c->chrConvertRange = ff_chrRangeToJpeg##bpc##_##opt; \ | |
| 463 | 461 | } \ |
| 464 | 462 | } while (0) |
| 465 | 463 | |
| 466 | −#define RANGE_CONVERT_FUNCS_DECL(opt) \ | |
| 467 | −void ff_lumRangeFromJpeg_ ##opt(int16_t *dst, int width, \ | |
| 468 | − uint32_t coeff, int64_t offset); \ | |
| 469 | −void ff_chrRangeFromJpeg_ ##opt(int16_t *dstU, int16_t *dstV, int width, \ | |
| 470 | − uint32_t coeff, int64_t offset); \ | |
| 471 | −void ff_lumRangeToJpeg_ ##opt(int16_t *dst, int width, \ | |
| 472 | − uint32_t coeff, int64_t offset); \ | |
| 473 | −void ff_chrRangeToJpeg_ ##opt(int16_t *dstU, int16_t *dstV, int width, \ | |
| 474 | − uint32_t coeff, int64_t offset); \ | |
| 475 | − | |
| 476 | −RANGE_CONVERT_FUNCS_DECL(sse2); | |
| 477 | −RANGE_CONVERT_FUNCS_DECL(avx2); | |
| 464 | +#define RANGE_CONVERT_FUNCS_DECL(opt, bpc) \ | |
| 465 | +void ff_lumRangeFromJpeg##bpc##_##opt(int16_t *dst, int width, \ | |
| 466 | + uint32_t coeff, int64_t offset); \ | |
| 467 | +void ff_chrRangeFromJpeg##bpc##_##opt(int16_t *dstU, int16_t *dstV, int width, \ | |
| 468 | + uint32_t coeff, int64_t offset); \ | |
| 469 | +void ff_lumRangeToJpeg##bpc##_##opt(int16_t *dst, int width, \ | |
| 470 | + uint32_t coeff, int64_t offset); \ | |
| 471 | +void ff_chrRangeToJpeg##bpc##_##opt(int16_t *dstU, int16_t *dstV, int width, \ | |
| 472 | + uint32_t coeff, int64_t offset); \ | |
| 473 | + | |
| 474 | +RANGE_CONVERT_FUNCS_DECL(sse2, 8) | |
| 475 | +RANGE_CONVERT_FUNCS_DECL(sse4, 16) | |
| 476 | +RANGE_CONVERT_FUNCS_DECL(avx2, 8) | |
| 477 | +RANGE_CONVERT_FUNCS_DECL(avx2, 16) | |
| 478 | 478 | |
| 479 | 479 | av_cold void ff_sws_init_range_convert_x86(SwsInternal *c) |
| 480 | 480 | { |
| 481 | 481 | int cpu_flags = av_get_cpu_flags(); |
| 482 | 482 | if (EXTERNAL_AVX2_FAST(cpu_flags)) { |
| 483 | − RANGE_CONVERT_FUNCS(avx2); | |
| 484 | − } else if (EXTERNAL_SSE2(cpu_flags)) { | |
| 485 | − RANGE_CONVERT_FUNCS(sse2); | |
| 483 | + if (c->dstBpc <= 14) { | |
| 484 | + RANGE_CONVERT_FUNCS(avx2, 8); | |
| 485 | + } else { | |
| 486 | + RANGE_CONVERT_FUNCS(avx2, 16); | |
| 487 | + } | |
| 488 | + } else if (EXTERNAL_SSE2(cpu_flags) && c->dstBpc <= 14) { | |
| 489 | + RANGE_CONVERT_FUNCS(sse2, 8); | |
| 490 | + } else if (EXTERNAL_SSE4(cpu_flags) && c->dstBpc > 14) { | |
| 491 | + RANGE_CONVERT_FUNCS(sse4, 16); | |
| 486 | 492 | } |
| 487 | 493 | } |
| 488 | 494 | |