checkasm --bench on a Raspberry Pi 5 Model B Rev 1.0: yuv420p_gbrp_128_c: 1243.0 yuv420p_gbrp_128_neon: 453.5 yuv420p_gbrp_1920_c: 18165.5 yuv420p_gbrp_1920_neon: 6700.0 yuv422p_gbrp_128_c: 1463.5 yuv422p_gbrp_128_neon: 471.5 yuv422p_gbrp_1920_c: 21343.7 yuv422p_gbrp_1920_neon: 6743.5
Showing the changes to libswscale (2 of 2 files). Show all
Unified · Split
| @@ -52,11 +52,41 @@ static int ifmt##_to_##ofmt##_neon_wrapper(SwsContext *c, const uint8_t *src[], · 51 unmodified lines | |||
| 52 | c->yuv2rgb_y_coeff); \ | 52 | c->yuv2rgb_y_coeff); \ |
| 53 | } \ | 53 | } \ |
| 54 | 54 | ||
| 55 | #define DECLARE_FF_YUVX_TO_GBRP_FUNCS(ifmt, ofmt) \ | ||
| 56 | int ff_##ifmt##_to_##ofmt##_neon(int w, int h, \ | ||
| 57 | uint8_t *dst, int linesize, \ | ||
| 58 | const uint8_t *srcY, int linesizeY, \ | ||
| 59 | const uint8_t *srcU, int linesizeU, \ | ||
| 60 | const uint8_t *srcV, int linesizeV, \ | ||
| 61 | const int16_t *table, \ | ||
| 62 | int y_offset, \ | ||
| 63 | int y_coeff, \ | ||
| 64 | uint8_t *dst1, int linesize1, \ | ||
| 65 | uint8_t *dst2, int linesize2); \ | ||
| 66 | \ | ||
| 67 | static int ifmt##_to_##ofmt##_neon_wrapper(SwsContext *c, const uint8_t *src[], \ | ||
| 68 | int srcStride[], int srcSliceY, int srcSliceH, \ | ||
| 69 | uint8_t *dst[], int dstStride[]) { \ | ||
| 70 | const int16_t yuv2rgb_table[] = { YUV_TO_RGB_TABLE }; \ | ||
| 71 | \ | ||
| 72 | return ff_##ifmt##_to_##ofmt##_neon(c->srcW, srcSliceH, \ | ||
| 73 | dst[0] + srcSliceY * dstStride[0], dstStride[0], \ | ||
| 74 | src[0], srcStride[0], \ | ||
| 75 | src[1], srcStride[1], \ | ||
| 76 | src[2], srcStride[2], \ | ||
| 77 | yuv2rgb_table, \ | ||
| 78 | c->yuv2rgb_y_offset >> 6, \ | ||
| 79 | c->yuv2rgb_y_coeff, \ | ||
| 80 | dst[1] + srcSliceY * dstStride[1], dstStride[1], \ | ||
| 81 | dst[2] + srcSliceY * dstStride[2], dstStride[2]); \ | ||
| 82 | } \ | ||
| 83 | |||
| 55 | #define DECLARE_FF_YUVX_TO_ALL_RGBX_FUNCS(yuvx) \ | 84 | #define DECLARE_FF_YUVX_TO_ALL_RGBX_FUNCS(yuvx) \ |
| 56 | DECLARE_FF_YUVX_TO_RGBX_FUNCS(yuvx, argb) \ | 85 | DECLARE_FF_YUVX_TO_RGBX_FUNCS(yuvx, argb) \ |
| 57 | DECLARE_FF_YUVX_TO_RGBX_FUNCS(yuvx, rgba) \ | 86 | DECLARE_FF_YUVX_TO_RGBX_FUNCS(yuvx, rgba) \ |
| 58 | DECLARE_FF_YUVX_TO_RGBX_FUNCS(yuvx, abgr) \ | 87 | DECLARE_FF_YUVX_TO_RGBX_FUNCS(yuvx, abgr) \ |
| 59 | DECLARE_FF_YUVX_TO_RGBX_FUNCS(yuvx, bgra) \ | 88 | DECLARE_FF_YUVX_TO_RGBX_FUNCS(yuvx, bgra) \ |
| 89 | DECLARE_FF_YUVX_TO_GBRP_FUNCS(yuvx, gbrp) \ | ||
| 60 | 90 | ||
| 61 | DECLARE_FF_YUVX_TO_ALL_RGBX_FUNCS(yuv420p) | 91 | DECLARE_FF_YUVX_TO_ALL_RGBX_FUNCS(yuv420p) |
| 62 | DECLARE_FF_YUVX_TO_ALL_RGBX_FUNCS(yuv422p) | 92 | DECLARE_FF_YUVX_TO_ALL_RGBX_FUNCS(yuv422p) |
| @@ -83,11 +113,38 @@ static int ifmt##_to_##ofmt##_neon_wrapper(SwsContext *c, const uint8_t *src[], · 20 unmodified lines | |||
| 83 | c->yuv2rgb_y_coeff); \ | 113 | c->yuv2rgb_y_coeff); \ |
| 84 | } \ | 114 | } \ |
| 85 | 115 | ||
| 116 | #define DECLARE_FF_NVX_TO_GBRP_FUNCS(ifmt, ofmt) \ | ||
| 117 | int ff_##ifmt##_to_##ofmt##_neon(int w, int h, \ | ||
| 118 | uint8_t *dst, int linesize, \ | ||
| 119 | const uint8_t *srcY, int linesizeY, \ | ||
| 120 | const uint8_t *srcC, int linesizeC, \ | ||
| 121 | const int16_t *table, \ | ||
| 122 | int y_offset, \ | ||
| 123 | int y_coeff, \ | ||
| 124 | uint8_t *dst1, int linesize1, \ | ||
| 125 | uint8_t *dst2, int linesize2); \ | ||
| 126 | \ | ||
| 127 | static int ifmt##_to_##ofmt##_neon_wrapper(SwsContext *c, const uint8_t *src[], \ | ||
| 128 | int srcStride[], int srcSliceY, int srcSliceH, \ | ||
| 129 | uint8_t *dst[], int dstStride[]) { \ | ||
| 130 | const int16_t yuv2rgb_table[] = { YUV_TO_RGB_TABLE }; \ | ||
| 131 | \ | ||
| 132 | return ff_##ifmt##_to_##ofmt##_neon(c->srcW, srcSliceH, \ | ||
| 133 | dst[0] + srcSliceY * dstStride[0], dstStride[0], \ | ||
| 134 | src[0], srcStride[0], src[1], srcStride[1], \ | ||
| 135 | yuv2rgb_table, \ | ||
| 136 | c->yuv2rgb_y_offset >> 6, \ | ||
| 137 | c->yuv2rgb_y_coeff, \ | ||
| 138 | dst[1] + srcSliceY * dstStride[1], dstStride[1], \ | ||
| 139 | dst[2] + srcSliceY * dstStride[2], dstStride[2]); \ | ||
| 140 | } \ | ||
| 141 | |||
| 86 | #define DECLARE_FF_NVX_TO_ALL_RGBX_FUNCS(nvx) \ | 142 | #define DECLARE_FF_NVX_TO_ALL_RGBX_FUNCS(nvx) \ |
| 87 | DECLARE_FF_NVX_TO_RGBX_FUNCS(nvx, argb) \ | 143 | DECLARE_FF_NVX_TO_RGBX_FUNCS(nvx, argb) \ |
| 88 | DECLARE_FF_NVX_TO_RGBX_FUNCS(nvx, rgba) \ | 144 | DECLARE_FF_NVX_TO_RGBX_FUNCS(nvx, rgba) \ |
| 89 | DECLARE_FF_NVX_TO_RGBX_FUNCS(nvx, abgr) \ | 145 | DECLARE_FF_NVX_TO_RGBX_FUNCS(nvx, abgr) \ |
| 90 | DECLARE_FF_NVX_TO_RGBX_FUNCS(nvx, bgra) \ | 146 | DECLARE_FF_NVX_TO_RGBX_FUNCS(nvx, bgra) \ |
| 147 | DECLARE_FF_NVX_TO_GBRP_FUNCS(nvx, gbrp) \ | ||
| 91 | 148 | ||
| 92 | DECLARE_FF_NVX_TO_ALL_RGBX_FUNCS(nv12) | 149 | DECLARE_FF_NVX_TO_ALL_RGBX_FUNCS(nv12) |
| 93 | DECLARE_FF_NVX_TO_ALL_RGBX_FUNCS(nv21) | 150 | DECLARE_FF_NVX_TO_ALL_RGBX_FUNCS(nv21) |
| @@ -110,6 +167,7 @@ DECLARE_FF_NVX_TO_ALL_RGBX_FUNCS(nv21) · 16 unmodified lines | |||
| 110 | SET_FF_NVX_TO_RGBX_FUNC(nvx, NVX, rgba, RGBA, accurate_rnd); \ | 167 | SET_FF_NVX_TO_RGBX_FUNC(nvx, NVX, rgba, RGBA, accurate_rnd); \ |
| 111 | SET_FF_NVX_TO_RGBX_FUNC(nvx, NVX, abgr, ABGR, accurate_rnd); \ | 168 | SET_FF_NVX_TO_RGBX_FUNC(nvx, NVX, abgr, ABGR, accurate_rnd); \ |
| 112 | SET_FF_NVX_TO_RGBX_FUNC(nvx, NVX, bgra, BGRA, accurate_rnd); \ | 169 | SET_FF_NVX_TO_RGBX_FUNC(nvx, NVX, bgra, BGRA, accurate_rnd); \ |
| 170 | SET_FF_NVX_TO_RGBX_FUNC(nvx, NVX, gbrp, GBRP, accurate_rnd); \ | ||
| 113 | } while (0) | 171 | } while (0) |
| 114 | 172 | ||
| 115 | static void get_unscaled_swscale_neon(SwsContext *c) { | 173 | static void get_unscaled_swscale_neon(SwsContext *c) { |
| @@ -30,23 +30,43 @@ · 29 unmodified lines | |||
| 30 | #endif | 30 | #endif |
| 31 | .endm | 31 | .endm |
| 32 | 32 | ||
| 33 | .macro load_args_nv12 | 33 | .macro load_dst1_dst2 dst1 linesize1 dst2 linesize2 |
| 34 | #if defined(__APPLE__) | ||
| 35 | #define DST_OFFSET 8 | ||
| 36 | #else | ||
| 37 | #define DST_OFFSET 0 | ||
| 38 | #endif | ||
| 39 | ldr x10, [sp, #\dst1 - DST_OFFSET] | ||
| 40 | ldr w12, [sp, #\linesize1 - DST_OFFSET] | ||
| 41 | ldr x15, [sp, #\dst2 - DST_OFFSET] | ||
| 42 | ldr w16, [sp, #\linesize2 - DST_OFFSET] | ||
| 43 | #undef DST_OFFSET | ||
| 44 | sub w12, w12, w0 // w12 = linesize1 - width (padding1) | ||
| 45 | sub w16, w16, w0 // w16 = linesize2 - width (padding2) | ||
| 46 | .endm | ||
| 47 | |||
| 48 | .macro load_args_nv12 ofmt | ||
| 34 | ldr x8, [sp] // table | 49 | ldr x8, [sp] // table |
| 35 | load_yoff_ycoeff 8, 16 // y_offset, y_coeff | 50 | load_yoff_ycoeff 8, 16 // y_offset, y_coeff |
| 36 | ld1 {v1.1d}, [x8] | 51 | ld1 {v1.1d}, [x8] |
| 37 | dup v0.8h, w10 | 52 | dup v0.8h, w10 |
| 38 | dup v3.8h, w9 | 53 | dup v3.8h, w9 |
| 54 | .ifc \ofmt,gbrp | ||
| 55 | load_dst1_dst2 24, 32, 40, 48 | ||
| 56 | sub w3, w3, w0 // w3 = linesize - width (padding) | ||
| 57 | .else | ||
| 39 | sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding) | 58 | sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding) |
| 59 | .endif | ||
| 40 | sub w5, w5, w0 // w5 = linesizeY - width (paddingY) | 60 | sub w5, w5, w0 // w5 = linesizeY - width (paddingY) |
| 41 | sub w7, w7, w0 // w7 = linesizeC - width (paddingC) | 61 | sub w7, w7, w0 // w7 = linesizeC - width (paddingC) |
| 42 | neg w11, w0 | 62 | neg w11, w0 |
| 43 | .endm | 63 | .endm |
| 44 | 64 | ||
| 45 | .macro load_args_nv21 | 65 | .macro load_args_nv21 ofmt |
| 46 | load_args_nv12 | 66 | load_args_nv12 \ofmt |
| 47 | .endm | 67 | .endm |
| 48 | 68 | ||
| 49 | .macro load_args_yuv420p | 69 | .macro load_args_yuv420p ofmt |
| 50 | ldr x13, [sp] // srcV | 70 | ldr x13, [sp] // srcV |
| 51 | ldr w14, [sp, #8] // linesizeV | 71 | ldr w14, [sp, #8] // linesizeV |
| 52 | ldr x8, [sp, #16] // table | 72 | ldr x8, [sp, #16] // table |
| @@ -54,7 +74,12 @@ · 1 unmodified line | |||
| 54 | ld1 {v1.1d}, [x8] | 74 | ld1 {v1.1d}, [x8] |
| 55 | dup v0.8h, w10 | 75 | dup v0.8h, w10 |
| 56 | dup v3.8h, w9 | 76 | dup v3.8h, w9 |
| 77 | .ifc \ofmt,gbrp | ||
| 78 | load_dst1_dst2 40, 48, 56, 64 | ||
| 79 | sub w3, w3, w0 // w3 = linesize - width (padding) | ||
| 80 | .else | ||
| 57 | sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding) | 81 | sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding) |
| 82 | .endif | ||
| 58 | sub w5, w5, w0 // w5 = linesizeY - width (paddingY) | 83 | sub w5, w5, w0 // w5 = linesizeY - width (paddingY) |
| 59 | sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU) | 84 | sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU) |
| 60 | sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV) | 85 | sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV) |
| @@ -62,7 +87,7 @@ · 1 unmodified line | |||
| 62 | neg w11, w11 | 87 | neg w11, w11 |
| 63 | .endm | 88 | .endm |
| 64 | 89 | ||
| 65 | .macro load_args_yuv422p | 90 | .macro load_args_yuv422p ofmt |
| 66 | ldr x13, [sp] // srcV | 91 | ldr x13, [sp] // srcV |
| 67 | ldr w14, [sp, #8] // linesizeV | 92 | ldr w14, [sp, #8] // linesizeV |
| 68 | ldr x8, [sp, #16] // table | 93 | ldr x8, [sp, #16] // table |
| @@ -70,7 +95,12 @@ · 1 unmodified line | |||
| 70 | ld1 {v1.1d}, [x8] | 95 | ld1 {v1.1d}, [x8] |
| 71 | dup v0.8h, w10 | 96 | dup v0.8h, w10 |
| 72 | dup v3.8h, w9 | 97 | dup v3.8h, w9 |
| 98 | .ifc \ofmt,gbrp | ||
| 99 | load_dst1_dst2 40, 48, 56, 64 | ||
| 100 | sub w3, w3, w0 // w3 = linesize - width (padding) | ||
| 101 | .else | ||
| 73 | sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding) | 102 | sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding) |
| 103 | .endif | ||
| 74 | sub w5, w5, w0 // w5 = linesizeY - width (paddingY) | 104 | sub w5, w5, w0 // w5 = linesizeY - width (paddingY) |
| 75 | sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU) | 105 | sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU) |
| 76 | sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV) | 106 | sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV) |
| @@ -100,9 +130,9 @@ · 23 unmodified lines | |||
| 100 | .endm | 130 | .endm |
| 101 | 131 | ||
| 102 | .macro increment_nv12 | 132 | .macro increment_nv12 |
| 103 | ands w15, w1, #1 | 133 | ands w17, w1, #1 |
| 104 | csel w16, w7, w11, ne // incC = (h & 1) ? paddincC : -width | 134 | csel w17, w7, w11, ne // incC = (h & 1) ? paddincC : -width |
| 105 | add x6, x6, w16, sxtw // srcC += incC | 135 | add x6, x6, w17, sxtw // srcC += incC |
| 106 | .endm | 136 | .endm |
| 107 | 137 | ||
| 108 | .macro increment_nv21 | 138 | .macro increment_nv21 |
| @@ -110,10 +140,10 @@ · 1 unmodified line | |||
| 110 | .endm | 140 | .endm |
| 111 | 141 | ||
| 112 | .macro increment_yuv420p | 142 | .macro increment_yuv420p |
| 113 | ands w15, w1, #1 | 143 | ands w17, w1, #1 |
| 114 | csel w16, w7, w11, ne // incU = (h & 1) ? paddincU : -width/2 | 144 | csel w17, w7, w11, ne // incU = (h & 1) ? paddincU : -width/2 |
| 145 | add x6, x6, w17, sxtw // srcU += incU | ||
| 115 | csel w17, w14, w11, ne // incV = (h & 1) ? paddincV : -width/2 | 146 | csel w17, w14, w11, ne // incV = (h & 1) ? paddincV : -width/2 |
| 116 | add x6, x6, w16, sxtw // srcU += incU | ||
| 117 | add x13, x13, w17, sxtw // srcV += incV | 147 | add x13, x13, w17, sxtw // srcV += incV |
| 118 | .endm | 148 | .endm |
| 119 | 149 | ||
| @@ -122,7 +152,7 @@ · 2 unmodified lines | |||
| 122 | add x13, x13, w14, sxtw // srcV += incV | 152 | add x13, x13, w14, sxtw // srcV += incV |
| 123 | .endm | 153 | .endm |
| 124 | 154 | ||
| 125 | .macro compute_rgba r1 g1 b1 a1 r2 g2 b2 a2 | 155 | .macro compute_rgb r1 g1 b1 r2 g2 b2 |
| 126 | add v20.8h, v26.8h, v20.8h // Y1 + R1 | 156 | add v20.8h, v26.8h, v20.8h // Y1 + R1 |
| 127 | add v21.8h, v27.8h, v21.8h // Y2 + R2 | 157 | add v21.8h, v27.8h, v21.8h // Y2 + R2 |
| 128 | add v22.8h, v26.8h, v22.8h // Y1 + G1 | 158 | add v22.8h, v26.8h, v22.8h // Y1 + G1 |
| @@ -135,13 +165,18 @@ · 6 unmodified lines | |||
| 135 | sqrshrun \g2, v23.8h, #1 // clip_u8((Y2 + G1) >> 1) | 165 | sqrshrun \g2, v23.8h, #1 // clip_u8((Y2 + G1) >> 1) |
| 136 | sqrshrun \b1, v24.8h, #1 // clip_u8((Y1 + B1) >> 1) | 166 | sqrshrun \b1, v24.8h, #1 // clip_u8((Y1 + B1) >> 1) |
| 137 | sqrshrun \b2, v25.8h, #1 // clip_u8((Y2 + B1) >> 1) | 167 | sqrshrun \b2, v25.8h, #1 // clip_u8((Y2 + B1) >> 1) |
| 168 | .endm | ||
| 169 | |||
| 170 | .macro compute_rgba r1 g1 b1 a1 r2 g2 b2 a2 | ||
| 171 | compute_rgb \r1, \g1, \b1, \r2, \g2, \b2 | ||
| 138 | movi \a1, #255 | 172 | movi \a1, #255 |
| 139 | movi \a2, #255 | 173 | movi \a2, #255 |
| 140 | .endm | 174 | .endm |
| 141 | 175 | ||
| 142 | .macro declare_func ifmt ofmt | 176 | .macro declare_func ifmt ofmt |
| 143 | function ff_\ifmt\()_to_\ofmt\()_neon, export=1 | 177 | function ff_\ifmt\()_to_\ofmt\()_neon, export=1 |
| 144 | load_args_\ifmt | 178 | load_args_\ifmt \ofmt |
| 179 | |||
| 145 | mov w9, w1 | 180 | mov w9, w1 |
| 146 | 1: | 181 | 1: |
| 147 | mov w8, w0 // w8 = width | 182 | mov w8, w0 // w8 = width |
| @@ -185,11 +220,22 @@ function ff_\ifmt\()_to_\ofmt\()_neon, export=1 · 37 unmodified lines | |||
| 185 | compute_rgba v6.8b,v5.8b,v4.8b,v7.8b, v18.8b,v17.8b,v16.8b,v19.8b | 220 | compute_rgba v6.8b,v5.8b,v4.8b,v7.8b, v18.8b,v17.8b,v16.8b,v19.8b |
| 186 | .endif | 221 | .endif |
| 187 | 222 | ||
| 223 | .ifc \ofmt,gbrp | ||
| 224 | compute_rgb v18.8b,v4.8b,v6.8b, v19.8b,v5.8b,v7.8b | ||
| 225 | st1 { v4.8b, v5.8b }, [x2], #16 | ||
| 226 | st1 { v6.8b, v7.8b }, [x10], #16 | ||
| 227 | st1 { v18.8b, v19.8b }, [x15], #16 | ||
| 228 | .else | ||
| 188 | st4 { v4.8b, v5.8b, v6.8b, v7.8b}, [x2], #32 | 229 | st4 { v4.8b, v5.8b, v6.8b, v7.8b}, [x2], #32 |
| 189 | st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [x2], #32 | 230 | st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [x2], #32 |
| 231 | .endif | ||
| 190 | subs w8, w8, #16 // width -= 16 | 232 | subs w8, w8, #16 // width -= 16 |
| 191 | b.gt 2b | 233 | b.gt 2b |
| 192 | add x2, x2, w3, sxtw // dst += padding | 234 | add x2, x2, w3, sxtw // dst += padding |
| 235 | .ifc \ofmt,gbrp | ||
| 236 | add x10, x10, w12, sxtw // dst1 += padding1 | ||
| 237 | add x15, x15, w16, sxtw // dst2 += padding2 | ||
| 238 | .endif | ||
| 193 | add x4, x4, w5, sxtw // srcY += paddingY | 239 | add x4, x4, w5, sxtw // srcY += paddingY |
| 194 | increment_\ifmt | 240 | increment_\ifmt |
| 195 | subs w1, w1, #1 // height -= 1 | 241 | subs w1, w1, #1 // height -= 1 |
| @@ -204,6 +250,7 @@ endfunc · 8 unmodified lines | |||
| 204 | declare_func \ifmt, rgba | 250 | declare_func \ifmt, rgba |
| 205 | declare_func \ifmt, abgr | 251 | declare_func \ifmt, abgr |
| 206 | declare_func \ifmt, bgra | 252 | declare_func \ifmt, bgra |
| 253 | declare_func \ifmt, gbrp | ||
| 207 | .endm | 254 | .endm |
| 208 | 255 | ||
| 209 | declare_rgb_funcs nv12 | 256 | declare_rgb_funcs nv12 |