chrRangeFromJpeg16_1920_c: 3153.9 chrRangeFromJpeg16_1920_sse4: 1770.0 (1.78x) chrRangeFromJpeg16_1920_avx2: 891.5 (3.54x) chrRangeToJpeg16_1920_c: 3165.0 chrRangeToJpeg16_1920_sse4: 1953.2 (1.62x) chrRangeToJpeg16_1920_avx2: 973.0 (3.25x) lumRangeFromJpeg16_1920_c: 1298.5 lumRangeFromJpeg16_1920_sse4: 886.5 (1.46x) lumRangeFromJpeg16_1920_avx2: 447.7 (2.90x) lumRangeToJpeg16_1920_c: 1905.0 lumRangeToJpeg16_1920_sse4: 993.0 (1.92x) lumRangeToJpeg16_1920_avx2: 498.9 (3.82x)
Unified · Split
| @@ -20,21 +20,24 @@ · 19 unmodified lines | |||
| 20 | 20 | ||
| 21 | %include "libavutil/x86/x86util.asm" | 21 | %include "libavutil/x86/x86util.asm" |
| 22 | 22 | ||
| 23 | SECTION_RODATA | ||
| 24 | pack19: times 4 dd (1 << 19) - 1 | ||
| 25 | |||
| 23 | SECTION .text | 26 | SECTION .text |
| 24 | 27 | ||
| 25 | ;----------------------------------------------------------------------------- | 28 | ;----------------------------------------------------------------------------- |
| 26 | ; lumConvertRange | 29 | ; lumConvertRange |
| 27 | ; | 30 | ; |
| 28 | ; void ff_lumRangeToJpeg_<opt>(int16_t *dst, int width, | 31 | ; void ff_lumRangeToJpeg{8,16}_<opt>(int16_t *dst, int width, |
| 29 | ; uint32_t coeff, int64_t offset); | 32 | ; uint32_t coeff, int64_t offset); |
| 30 | ; void ff_lumRangeFromJpeg_<opt>(int16_t *dst, int width, | 33 | ; void ff_lumRangeFromJpeg{8,16}_<opt>(int16_t *dst, int width, |
| 31 | ; uint32_t coeff, int64_t offset); | 34 | ; uint32_t coeff, int64_t offset); |
| 32 | ; | 35 | ; |
| 33 | ;----------------------------------------------------------------------------- | 36 | ;----------------------------------------------------------------------------- |
| 34 | 37 | ||
| 35 | %macro LUMCONVERTRANGE 1 | 38 | %macro LUMCONVERTRANGE 2 |
| 36 | cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset | 39 | cglobal lumRange%1Jpeg%2, 4, 4, 5, dst, width, coeff, offset |
| 37 | shl widthd, 1 | 40 | shl widthd, %2 >> 3 |
| 38 | movd xm2, coeffd | 41 | movd xm2, coeffd |
| 39 | VBROADCASTSS m2, xm2 | 42 | VBROADCASTSS m2, xm2 |
| 40 | %if ARCH_X86_64 | 43 | %if ARCH_X86_64 |
| @@ -42,12 +45,34 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 1 unmodified line | |||
| 42 | %else | 45 | %else |
| 43 | movq xm3, offsetm | 46 | movq xm3, offsetm |
| 44 | %endif | 47 | %endif |
| 48 | %if %2 == 16 | ||
| 49 | VBROADCASTSD m3, xm3 | ||
| 50 | %ifidni %1,To | ||
| 51 | VBROADCASTI128 m4, [pack19] | ||
| 52 | %endif | ||
| 53 | %elif %2 == 8 | ||
| 45 | VBROADCASTSS m3, xm3 | 54 | VBROADCASTSS m3, xm3 |
| 46 | pxor m4, m4 | 55 | pxor m4, m4 |
| 56 | %endif ; %2 == 8/16 | ||
| 47 | add dstq, widthq | 57 | add dstq, widthq |
| 48 | neg widthq | 58 | neg widthq |
| 49 | .loop: | 59 | .loop: |
| 50 | movu m0, [dstq+widthq] | 60 | movu m0, [dstq+widthq] |
| 61 | %if %2 == 16 | ||
| 62 | pshufd m1, m0, 0xb1 | ||
| 63 | pmuldq m0, m2 | ||
| 64 | pmuldq m1, m2 | ||
| 65 | paddq m0, m3 | ||
| 66 | paddq m1, m3 | ||
| 67 | psrlq m0, 18 | ||
| 68 | psrlq m1, 18 | ||
| 69 | pshufd m0, m0, 0xd8 | ||
| 70 | pshufd m1, m1, 0xd8 | ||
| 71 | punpckldq m0, m1 | ||
| 72 | %ifidni %1,To | ||
| 73 | PMINSD m0, m4, m1 | ||
| 74 | %endif | ||
| 75 | %elif %2 == 8 | ||
| 51 | punpckhwd m1, m0, m4 | 76 | punpckhwd m1, m0, m4 |
| 52 | punpcklwd m0, m4 | 77 | punpcklwd m0, m4 |
| 53 | pmaddwd m0, m2 | 78 | pmaddwd m0, m2 |
| @@ -57,6 +82,7 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 3 unmodified lines | |||
| 57 | psrad m0, 14 | 82 | psrad m0, 14 |
| 58 | psrad m1, 14 | 83 | psrad m1, 14 |
| 59 | packssdw m0, m1 | 84 | packssdw m0, m1 |
| 85 | %endif ; %2 == 8/16 | ||
| 60 | movu [dstq+widthq], m0 | 86 | movu [dstq+widthq], m0 |
| 61 | add widthq, mmsize | 87 | add widthq, mmsize |
| 62 | jl .loop | 88 | jl .loop |
| @@ -66,16 +92,16 @@ cglobal lumRange%1Jpeg, 4, 4, 5, dst, width, coeff, offset · 3 unmodified lines | |||
| 66 | ;----------------------------------------------------------------------------- | 92 | ;----------------------------------------------------------------------------- |
| 67 | ; chrConvertRange | 93 | ; chrConvertRange |
| 68 | ; | 94 | ; |
| 69 | ; void ff_chrRangeToJpeg_<opt>(int16_t *dstU, int16_t *dstV, int width, | 95 | ; void ff_chrRangeToJpeg{8,16}_<opt>(int16_t *dstU, int16_t *dstV, int width, |
| 70 | ; uint32_t coeff, int64_t offset); | 96 | ; uint32_t coeff, int64_t offset); |
| 71 | ; void ff_chrRangeFromJpeg_<opt>(int16_t *dstU, int16_t *dstV, int width, | 97 | ; void ff_chrRangeFromJpeg{8,16}_<opt>(int16_t *dstU, int16_t *dstV, int width, |
| 72 | ; uint32_t coeff, int64_t offset); | 98 | ; uint32_t coeff, int64_t offset); |
| 73 | ; | 99 | ; |
| 74 | ;----------------------------------------------------------------------------- | 100 | ;----------------------------------------------------------------------------- |
| 75 | 101 | ||
| 76 | %macro CHRCONVERTRANGE 1 | 102 | %macro CHRCONVERTRANGE 2 |
| 77 | cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset | 103 | cglobal chrRange%1Jpeg%2, 5, 5, 7, dstU, dstV, width, coeff, offset |
| 78 | shl widthd, 1 | 104 | shl widthd, %2 >> 3 |
| 79 | movd xm4, coeffd | 105 | movd xm4, coeffd |
| 80 | VBROADCASTSS m4, xm4 | 106 | VBROADCASTSS m4, xm4 |
| 81 | %if ARCH_X86_64 | 107 | %if ARCH_X86_64 |
| @@ -83,14 +109,47 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 1 unmodified line | |||
| 83 | %else | 109 | %else |
| 84 | movq xm5, offsetm | 110 | movq xm5, offsetm |
| 85 | %endif | 111 | %endif |
| 112 | %if %2 == 16 | ||
| 113 | VBROADCASTSD m5, xm5 | ||
| 114 | %ifidni %1,To | ||
| 115 | VBROADCASTI128 m6, [pack19] | ||
| 116 | %endif | ||
| 117 | %elif %2 == 8 | ||
| 86 | VBROADCASTSS m5, xm5 | 118 | VBROADCASTSS m5, xm5 |
| 87 | pxor m6, m6 | 119 | pxor m6, m6 |
| 120 | %endif ; %2 == 8/16 | ||
| 88 | add dstUq, widthq | 121 | add dstUq, widthq |
| 89 | add dstVq, widthq | 122 | add dstVq, widthq |
| 90 | neg widthq | 123 | neg widthq |
| 91 | .loop: | 124 | .loop: |
| 92 | movu m0, [dstUq+widthq] | 125 | movu m0, [dstUq+widthq] |
| 93 | movu m2, [dstVq+widthq] | 126 | movu m2, [dstVq+widthq] |
| 127 | %if %2 == 16 | ||
| 128 | pshufd m1, m0, 0xb1 | ||
| 129 | pshufd m3, m2, 0xb1 | ||
| 130 | pmuldq m0, m4 | ||
| 131 | pmuldq m1, m4 | ||
| 132 | pmuldq m2, m4 | ||
| 133 | pmuldq m3, m4 | ||
| 134 | paddq m0, m5 | ||
| 135 | paddq m1, m5 | ||
| 136 | paddq m2, m5 | ||
| 137 | paddq m3, m5 | ||
| 138 | psrlq m0, 18 | ||
| 139 | psrlq m1, 18 | ||
| 140 | psrlq m2, 18 | ||
| 141 | psrlq m3, 18 | ||
| 142 | pshufd m0, m0, 0xd8 | ||
| 143 | pshufd m1, m1, 0xd8 | ||
| 144 | pshufd m2, m2, 0xd8 | ||
| 145 | pshufd m3, m3, 0xd8 | ||
| 146 | punpckldq m0, m1 | ||
| 147 | punpckldq m2, m3 | ||
| 148 | %ifidni %1,To | ||
| 149 | PMINSD m0, m6, m1 | ||
| 150 | PMINSD m2, m6, m3 | ||
| 151 | %endif | ||
| 152 | %elif %2 == 8 | ||
| 94 | punpckhwd m1, m0, m6 | 153 | punpckhwd m1, m0, m6 |
| 95 | punpckhwd m3, m2, m6 | 154 | punpckhwd m3, m2, m6 |
| 96 | punpcklwd m0, m6 | 155 | punpcklwd m0, m6 |
| @@ -109,6 +168,7 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 12 unmodified lines | |||
| 109 | psrad m3, 14 | 168 | psrad m3, 14 |
| 110 | packssdw m0, m1 | 169 | packssdw m0, m1 |
| 111 | packssdw m2, m3 | 170 | packssdw m2, m3 |
| 171 | %endif ; %2 == 8/16 | ||
| 112 | movu [dstUq+widthq], m0 | 172 | movu [dstUq+widthq], m0 |
| 113 | movu [dstVq+widthq], m2 | 173 | movu [dstVq+widthq], m2 |
| 114 | add widthq, mmsize | 174 | add widthq, mmsize |
| @@ -117,15 +177,25 @@ cglobal chrRange%1Jpeg, 5, 5, 7, dstU, dstV, width, coeff, offset · 2 unmodified lines | |||
| 117 | %endmacro | 177 | %endmacro |
| 118 | 178 | ||
| 119 | INIT_XMM sse2 | 179 | INIT_XMM sse2 |
| 120 | LUMCONVERTRANGE To | 180 | LUMCONVERTRANGE To, 8 |
| 121 | CHRCONVERTRANGE To | 181 | CHRCONVERTRANGE To, 8 |
| 122 | LUMCONVERTRANGE From | 182 | LUMCONVERTRANGE From, 8 |
| 123 | CHRCONVERTRANGE From | 183 | CHRCONVERTRANGE From, 8 |
| 184 | |||
| 185 | INIT_XMM sse4 | ||
| 186 | LUMCONVERTRANGE To, 16 | ||
| 187 | CHRCONVERTRANGE To, 16 | ||
| 188 | LUMCONVERTRANGE From, 16 | ||
| 189 | CHRCONVERTRANGE From, 16 | ||
| 124 | 190 | ||
| 125 | %if HAVE_AVX2_EXTERNAL | 191 | %if HAVE_AVX2_EXTERNAL |
| 126 | INIT_YMM avx2 | 192 | INIT_YMM avx2 |
| 127 | LUMCONVERTRANGE To | 193 | LUMCONVERTRANGE To, 8 |
| 128 | CHRCONVERTRANGE To | 194 | LUMCONVERTRANGE To, 16 |
| 129 | LUMCONVERTRANGE From | 195 | CHRCONVERTRANGE To, 8 |
| 130 | CHRCONVERTRANGE From | 196 | CHRCONVERTRANGE To, 16 |
| 197 | LUMCONVERTRANGE From, 8 | ||
| 198 | LUMCONVERTRANGE From, 16 | ||
| 199 | CHRCONVERTRANGE From, 8 | ||
| 200 | CHRCONVERTRANGE From, 16 | ||
| 131 | %endif | 201 | %endif |
| @@ -451,38 +451,44 @@ INPUT_PLANAR_RGB_UV_ALL_DECL(avx2); · 450 unmodified lines | |||
| 451 | INPUT_PLANAR_RGB_A_ALL_DECL(avx2); | 451 | INPUT_PLANAR_RGB_A_ALL_DECL(avx2); |
| 452 | #endif | 452 | #endif |
| 453 | 453 | ||
| 454 | #define RANGE_CONVERT_FUNCS(opt) do { \ | 454 | #define RANGE_CONVERT_FUNCS(opt, bpc) do { \ |
| 455 | if (c->dstBpc <= 14) { \ | 455 | if (c->opts.src_range) { \ |
| 456 | if (c->opts.src_range) { \ | 456 | c->lumConvertRange = ff_lumRangeFromJpeg##bpc##_##opt; \ |
| 457 | c->lumConvertRange = ff_lumRangeFromJpeg_ ##opt; \ | 457 | c->chrConvertRange = ff_chrRangeFromJpeg##bpc##_##opt; \ |
| 458 | c->chrConvertRange = ff_chrRangeFromJpeg_ ##opt; \ | 458 | } else { \ |
| 459 | } else { \ | 459 | c->lumConvertRange = ff_lumRangeToJpeg##bpc##_##opt; \ |
| 460 | c->lumConvertRange = ff_lumRangeToJpeg_ ##opt; \ | 460 | c->chrConvertRange = ff_chrRangeToJpeg##bpc##_##opt; \ |
| 461 | c->chrConvertRange = ff_chrRangeToJpeg_ ##opt; \ | ||
| 462 | } \ | ||
| 463 | } \ | 461 | } \ |
| 464 | } while (0) | 462 | } while (0) |
| 465 | 463 | ||
| 466 | #define RANGE_CONVERT_FUNCS_DECL(opt) \ | 464 | #define RANGE_CONVERT_FUNCS_DECL(opt, bpc) \ |
| 467 | void ff_lumRangeFromJpeg_ ##opt(int16_t *dst, int width, \ | 465 | void ff_lumRangeFromJpeg##bpc##_##opt(int16_t *dst, int width, \ |
| 468 | uint32_t coeff, int64_t offset); \ | 466 | uint32_t coeff, int64_t offset); \ |
| 469 | void ff_chrRangeFromJpeg_ ##opt(int16_t *dstU, int16_t *dstV, int width, \ | 467 | void ff_chrRangeFromJpeg##bpc##_##opt(int16_t *dstU, int16_t *dstV, int width, \ |
| 470 | uint32_t coeff, int64_t offset); \ | 468 | uint32_t coeff, int64_t offset); \ |
| 471 | void ff_lumRangeToJpeg_ ##opt(int16_t *dst, int width, \ | 469 | void ff_lumRangeToJpeg##bpc##_##opt(int16_t *dst, int width, \ |
| 472 | uint32_t coeff, int64_t offset); \ | 470 | uint32_t coeff, int64_t offset); \ |
| 473 | void ff_chrRangeToJpeg_ ##opt(int16_t *dstU, int16_t *dstV, int width, \ | 471 | void ff_chrRangeToJpeg##bpc##_##opt(int16_t *dstU, int16_t *dstV, int width, \ |
| 474 | uint32_t coeff, int64_t offset); \ | 472 | uint32_t coeff, int64_t offset); \ |
| 475 | 473 | ||
| 476 | RANGE_CONVERT_FUNCS_DECL(sse2); | 474 | RANGE_CONVERT_FUNCS_DECL(sse2, 8) |
| 477 | RANGE_CONVERT_FUNCS_DECL(avx2); | 475 | RANGE_CONVERT_FUNCS_DECL(sse4, 16) |
| 476 | RANGE_CONVERT_FUNCS_DECL(avx2, 8) | ||
| 477 | RANGE_CONVERT_FUNCS_DECL(avx2, 16) | ||
| 478 | 478 | ||
| 479 | av_cold void ff_sws_init_range_convert_x86(SwsInternal *c) | 479 | av_cold void ff_sws_init_range_convert_x86(SwsInternal *c) |
| 480 | { | 480 | { |
| 481 | int cpu_flags = av_get_cpu_flags(); | 481 | int cpu_flags = av_get_cpu_flags(); |
| 482 | if (EXTERNAL_AVX2_FAST(cpu_flags)) { | 482 | if (EXTERNAL_AVX2_FAST(cpu_flags)) { |
| 483 | RANGE_CONVERT_FUNCS(avx2); | 483 | if (c->dstBpc <= 14) { |
| 484 | } else if (EXTERNAL_SSE2(cpu_flags)) { | 484 | RANGE_CONVERT_FUNCS(avx2, 8); |
| 485 | RANGE_CONVERT_FUNCS(sse2); | 485 | } else { |
| 486 | RANGE_CONVERT_FUNCS(avx2, 16); | ||
| 487 | } | ||
| 488 | } else if (EXTERNAL_SSE2(cpu_flags) && c->dstBpc <= 14) { | ||
| 489 | RANGE_CONVERT_FUNCS(sse2, 8); | ||
| 490 | } else if (EXTERNAL_SSE4(cpu_flags) && c->dstBpc > 14) { | ||
| 491 | RANGE_CONVERT_FUNCS(sse4, 16); | ||
| 486 | } | 492 | } |
| 487 | } | 493 | } |
| 488 | 494 | ||