History

swscale/aarch64: dotprod implementation of rgba32_to_Y

The idea is to split the 16 bit coefficients into lower and upper half, invoke udot for the lower half, shift by 8, and follow by udot for the upper half. Benchmark on A78: bgra_to_y_128_c: 682.0 ( 1.00x) bgra_to_y_128_neon: 181.2 ( 3.76x) bgra_to_y_128_dotprod: 117.8 ( 5.79x) bgra_to_y_1080_c: 5742.5 ( 1.00x) bgra_to_y_1080_neon: 1472.5 ( 3.90x) bgra_to_y_1080_dotprod: 906.5 ( 6.33x) bgra_to_y_1920_c: 10194.0 ( 1.00x) bgra_to_y_1920_neon: 2589.8 ( 3.94x) bgra_to_y_1920_dotprod: 1573.8 ( 6.48x)

Krzysztof Pyrkosz committed 1 year ago

Commit d765e5f1 parent 081c8652 files · +105 −0Browse files

Unified · Split

2 files
  1. libswscale/aarch64/input.S +88 −0
  2. libswscale/aarch64/swscale.c +17 −0
libswscale/aarch64/input.S+88−0
@@ -313,3 +313,91 @@ rgbToUV_neon bgr24, rgb24, element=3 · 312 unmodified lines
313rgbToUV_neon bgra32, rgba32, element=4313rgbToUV_neon bgra32, rgba32, element=4
314 314
315rgbToUV_neon abgr32, argb32, element=4, alpha_first=1315rgbToUV_neon abgr32, argb32, element=4, alpha_first=1
316
317#if HAVE_DOTPROD
318ENABLE_DOTPROD
319
320function ff_bgra32ToY_neon_dotprod, export=1
321 cmp w4, #0 // check width > 0
322 ldp w12, w11, [x5] // w12: ry, w11: gy
323 ldr w10, [x5, #8] // w10: by
324 b.gt 4f
325 ret
326endfunc
327
328function ff_rgba32ToY_neon_dotprod, export=1
329 cmp w4, #0 // check width > 0
330 ldp w10, w11, [x5] // w10: ry, w11: gy
331 ldr w12, [x5, #8] // w12: by
332 b.le 3f
3334:
334 mov w9, #256 // w9 = 1 << (RGB2YUV_SHIFT - 7)
335 movk w9, #8, lsl #16 // w9 += 32 << (RGB2YUV_SHIFT - 1)
336 dup v6.4s, w9 // w9: const_offset
337
338 cmp w4, #16
339 mov w7, w10
340 bfi w7, w11, 8, 8 // the bfi instructions are used to assemble
341 bfi w7, w12, 16, 8 // 4 byte r,g,b,0 mask to be then used by udot.
342 dup v0.4s, w7 // v0 holds the lower byte of each coefficient
343
344 lsr w6, w10, #8
345 lsr w7, w11, #8
346 lsr w8, w12, #8
347
348 bfi w6, w7, 8, 8
349 bfi w6, w8, 16, 8
350 dup v1.4s, w6 // v1 holds the upper byte of each coefficient
351 b.lt 2f
3521:
353 ld1 { v16.16b, v17.16b, v18.16b, v19.16b }, [x1], #64
354 sub w4, w4, #16 // width -= 16
355
356 mov v2.16b, v6.16b
357 mov v3.16b, v6.16b
358 mov v4.16b, v6.16b
359 mov v5.16b, v6.16b
360 cmp w4, #16 // width >= 16 ?
361
362 udot v2.4s, v16.16b, v0.16b
363 udot v3.4s, v17.16b, v0.16b
364 udot v4.4s, v18.16b, v0.16b
365 udot v5.4s, v19.16b, v0.16b
366
367 ushr v2.4s, v2.4s, #8
368 ushr v3.4s, v3.4s, #8
369 ushr v4.4s, v4.4s, #8
370 ushr v5.4s, v5.4s, #8
371
372 udot v2.4s, v16.16b, v1.16b
373 udot v3.4s, v17.16b, v1.16b
374 udot v4.4s, v18.16b, v1.16b
375 udot v5.4s, v19.16b, v1.16b
376
377 sqshrn v16.4h, v2.4s, #1
378 sqshrn2 v16.8h, v3.4s, #1
379 sqshrn v17.4h, v4.4s, #1
380 sqshrn2 v17.8h, v5.4s, #1
381
382 stp q16, q17, [x0], #32 // store to dst
383 b.ge 1b
384 cbz x4, 3f
3852:
386 ldrb w13, [x1] // w13: r
387 ldrb w14, [x1, #1] // w14: g
388 ldrb w15, [x1, #2] // w15: b
389
390 smaddl x13, w13, w10, x9 // x13 = ry * r + const_offset
391 smaddl x13, w14, w11, x13 // x13 += gy * g
392 smaddl x13, w15, w12, x13 // x13 += by * b
393 asr w13, w13, #9 // x13 >>= 9
394 sub w4, w4, #1 // width--
395 add x1, x1, #4
396 strh w13, [x0], #2 // store to dst
397 cbnz w4, 2b
3983:
399 ret
400endfunc
401
402DISABLE_DOTPROD
403#endif
libswscale/aarch64/swscale.c+17−0
@@ -210,6 +210,9 @@ void ff_##name##ToUV_neon(uint8_t *, uint8_t *, const uint8_t *, \ · 209 unmodified lines
210void ff_##name##ToUV_half_neon(uint8_t *, uint8_t *, const uint8_t *, \210void ff_##name##ToUV_half_neon(uint8_t *, uint8_t *, const uint8_t *, \
211 const uint8_t *, const uint8_t *, int w, \211 const uint8_t *, const uint8_t *, int w, \
212 uint32_t *coeffs, void *)212 uint32_t *coeffs, void *)
213#define NEON_INPUT_DOTPROD(name) \
214void ff_##name##ToY_neon_dotprod(uint8_t *dst, const uint8_t *src, const uint8_t *, \
215 const uint8_t *, int w, uint32_t *coeffs, void *);
213 216
214NEON_INPUT(abgr32);217NEON_INPUT(abgr32);
215NEON_INPUT(argb32);218NEON_INPUT(argb32);
@@ -217,6 +220,8 @@ NEON_INPUT(bgr24); · 1 unmodified line
217NEON_INPUT(bgra32);220NEON_INPUT(bgra32);
218NEON_INPUT(rgb24);221NEON_INPUT(rgb24);
219NEON_INPUT(rgba32);222NEON_INPUT(rgba32);
223NEON_INPUT_DOTPROD(bgra32);
224NEON_INPUT_DOTPROD(rgba32);
220 225
221void ff_lumRangeFromJpeg8_neon(int16_t *dst, int width,226void ff_lumRangeFromJpeg8_neon(int16_t *dst, int width,
222 uint32_t coeff, int64_t offset);227 uint32_t coeff, int64_t offset);
@@ -295,6 +300,12 @@ av_cold void ff_sws_init_swscale_aarch64(SwsInternal *c) · 72 unmodified lines
295 c->chrToYV12 = ff_bgr24ToUV_neon;300 c->chrToYV12 = ff_bgr24ToUV_neon;
296 break;301 break;
297 case AV_PIX_FMT_BGRA:302 case AV_PIX_FMT_BGRA:
303#if HAVE_DOTPROD
304 if (have_dotprod(cpu_flags)) {
305 c->lumToYV12 = ff_bgra32ToY_neon_dotprod;
306 }
307 else
308#endif
298 c->lumToYV12 = ff_bgra32ToY_neon;309 c->lumToYV12 = ff_bgra32ToY_neon;
299 if (c->chrSrcHSubSample)310 if (c->chrSrcHSubSample)
300 c->chrToYV12 = ff_bgra32ToUV_half_neon;311 c->chrToYV12 = ff_bgra32ToUV_half_neon;
@@ -309,6 +320,12 @@ av_cold void ff_sws_init_swscale_aarch64(SwsInternal *c) · 8 unmodified lines
309 c->chrToYV12 = ff_rgb24ToUV_neon;320 c->chrToYV12 = ff_rgb24ToUV_neon;
310 break;321 break;
311 case AV_PIX_FMT_RGBA:322 case AV_PIX_FMT_RGBA:
323#if HAVE_DOTPROD
324 if (have_dotprod(cpu_flags)) {
325 c->lumToYV12 = ff_rgba32ToY_neon_dotprod;
326 }
327 else
328#endif
312 c->lumToYV12 = ff_rgba32ToY_neon;329 c->lumToYV12 = ff_rgba32ToY_neon;
313 if (c->chrSrcHSubSample)330 if (c->chrSrcHSubSample)
314 c->chrToYV12 = ff_rgba32ToUV_half_neon;331 c->chrToYV12 = ff_rgba32ToUV_half_neon;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t