checkasm --bench for Raspberry Pi 5 Model B Rev 1.0: sse_0_c: 241.5 sse_0_neon: 37.2 sse_0_dotprod: 22.2 vsse_4_c: 148.7 vsse_4_neon: 31.0 vsse_4_dotprod: 15.7
Showing the changes to libavcodec (2 of 2 files). Show all
Unified · Split
| @@ -77,6 +77,13 @@ int vsse8_neon(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2, · 76 unmodified lines | ||
| 77 | 77 | int vsse_intra8_neon(MpegEncContext *c, const uint8_t *s, const uint8_t *dummy, |
| 78 | 78 | ptrdiff_t stride, int h); |
| 79 | 79 | |
| 80 | +#if HAVE_DOTPROD | |
| 81 | +int sse16_neon_dotprod(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2, | |
| 82 | + ptrdiff_t stride, int h); | |
| 83 | +int vsse_intra16_neon_dotprod(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2, | |
| 84 | + ptrdiff_t stride, int h); | |
| 85 | +#endif | |
| 86 | + | |
| 80 | 87 | av_cold void ff_me_cmp_init_aarch64(MECmpContext *c, AVCodecContext *avctx) |
| 81 | 88 | { |
| 82 | 89 | int cpu_flags = av_get_cpu_flags(); |
| @@ -113,6 +120,13 @@ av_cold void ff_me_cmp_init_aarch64(MECmpContext *c, AVCodecContext *avctx) · 30 unmodified lines | ||
| 113 | 120 | c->median_sad[0] = pix_median_abs16_neon; |
| 114 | 121 | c->median_sad[1] = pix_median_abs8_neon; |
| 115 | 122 | } |
| 123 | + | |
| 124 | +#if HAVE_DOTPROD | |
| 125 | + if (have_dotprod(cpu_flags)) { | |
| 126 | + c->sse[0] = sse16_neon_dotprod; | |
| 127 | + c->vsse[4] = vsse_intra16_neon_dotprod; | |
| 128 | + } | |
| 129 | +#endif | |
| 116 | 130 | } |
| 117 | 131 | |
| 118 | 132 | int nsse16_neon_wrapper(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2, |
| @@ -1559,3 +1559,117 @@ function pix_median_abs8_neon, export=1 · 1558 unmodified lines | ||
| 1559 | 1559 | ret |
| 1560 | 1560 | |
| 1561 | 1561 | endfunc |
| 1562 | + | |
| 1563 | +#if HAVE_DOTPROD | |
| 1564 | +ENABLE_DOTPROD | |
| 1565 | + | |
| 1566 | +function sse16_neon_dotprod, export=1 | |
| 1567 | + // x0 - unused | |
| 1568 | + // x1 - pix1 | |
| 1569 | + // x2 - pix2 | |
| 1570 | + // x3 - stride | |
| 1571 | + // w4 - h | |
| 1572 | + | |
| 1573 | + cmp w4, #4 | |
| 1574 | + movi v17.4s, #0 | |
| 1575 | + b.lt 2f | |
| 1576 | + | |
| 1577 | +// Make 4 iterations at once | |
| 1578 | +1: | |
| 1579 | + | |
| 1580 | + // res = abs(pix1[0] - pix2[0]) | |
| 1581 | + // res * res | |
| 1582 | + | |
| 1583 | + ld1 {v0.16b}, [x1], x3 // Load pix1 vector for first iteration | |
| 1584 | + ld1 {v1.16b}, [x2], x3 // Load pix2 vector for first iteration | |
| 1585 | + ld1 {v2.16b}, [x1], x3 // Load pix1 vector for second iteration | |
| 1586 | + uabd v30.16b, v0.16b, v1.16b // Absolute difference, first iteration | |
| 1587 | + ld1 {v3.16b}, [x2], x3 // Load pix2 vector for second iteration | |
| 1588 | + udot v17.4s, v30.16b, v30.16b | |
| 1589 | + uabd v27.16b, v2.16b, v3.16b // Absolute difference, second iteration | |
| 1590 | + ld1 {v4.16b}, [x1], x3 // Load pix1 for third iteration | |
| 1591 | + udot v17.4s, v27.16b, v27.16b | |
| 1592 | + ld1 {v5.16b}, [x2], x3 // Load pix2 for third iteration | |
| 1593 | + uabd v24.16b, v4.16b, v5.16b // Absolute difference, third iteration | |
| 1594 | + ld1 {v6.16b}, [x1], x3 // Load pix1 for fourth iteration | |
| 1595 | + udot v17.4s, v24.16b, v24.16b | |
| 1596 | + ld1 {v7.16b}, [x2], x3 // Load pix2 for fouth iteration | |
| 1597 | + uabd v21.16b, v6.16b, v7.16b // Absolute difference, fourth iteration | |
| 1598 | + sub w4, w4, #4 // h -= 4 | |
| 1599 | + udot v17.4s, v21.16b, v21.16b | |
| 1600 | + cmp w4, #4 | |
| 1601 | + b.ge 1b | |
| 1602 | + | |
| 1603 | + cbz w4, 3f | |
| 1604 | + | |
| 1605 | +// iterate by one | |
| 1606 | +2: | |
| 1607 | + | |
| 1608 | + ld1 {v0.16b}, [x1], x3 // Load pix1 | |
| 1609 | + ld1 {v1.16b}, [x2], x3 // Load pix2 | |
| 1610 | + | |
| 1611 | + uabd v30.16b, v0.16b, v1.16b | |
| 1612 | + subs w4, w4, #1 | |
| 1613 | + udot v17.4s, v30.16b, v30.16b | |
| 1614 | + | |
| 1615 | + b.ne 2b | |
| 1616 | + | |
| 1617 | +3: | |
| 1618 | + uaddlv d16, v17.4s // add up accumulator vector | |
| 1619 | + | |
| 1620 | + fmov w0, s16 | |
| 1621 | + | |
| 1622 | + ret | |
| 1623 | +endfunc | |
| 1624 | + | |
| 1625 | +function vsse_intra16_neon_dotprod, export=1 | |
| 1626 | + // x0 unused | |
| 1627 | + // x1 uint8_t *pix1 | |
| 1628 | + // x2 uint8_t *dummy | |
| 1629 | + // x3 ptrdiff_t stride | |
| 1630 | + // w4 int h | |
| 1631 | + | |
| 1632 | + ld1 {v0.16b}, [x1], x3 | |
| 1633 | + movi v17.4s, #0 | |
| 1634 | + | |
| 1635 | + sub w4, w4, #1 // we need to make h-1 iterations | |
| 1636 | + cmp w4, #3 | |
| 1637 | + b.lt 2f | |
| 1638 | + | |
| 1639 | +1: | |
| 1640 | + // v = abs( pix1[0] - pix1[0 + stride] ) | |
| 1641 | + // score = sum( v * v ) | |
| 1642 | + ld1 {v1.16b}, [x1], x3 | |
| 1643 | + ld1 {v2.16b}, [x1], x3 | |
| 1644 | + uabd v30.16b, v0.16b, v1.16b | |
| 1645 | + ld1 {v3.16b}, [x1], x3 | |
| 1646 | + udot v17.4s, v30.16b, v30.16b | |
| 1647 | + uabd v27.16b, v1.16b, v2.16b | |
| 1648 | + udot v17.4s, v27.16b, v27.16b | |
| 1649 | + uabd v25.16b, v2.16b, v3.16b | |
| 1650 | + sub w4, w4, #3 | |
| 1651 | + udot v17.4s, v25.16b, v25.16b | |
| 1652 | + cmp w4, #3 | |
| 1653 | + mov v0.16b, v3.16b | |
| 1654 | + | |
| 1655 | + b.ge 1b | |
| 1656 | + cbz w4, 3f | |
| 1657 | + | |
| 1658 | +// iterate by one | |
| 1659 | +2: | |
| 1660 | + ld1 {v1.16b}, [x1], x3 | |
| 1661 | + subs w4, w4, #1 | |
| 1662 | + uabd v30.16b, v0.16b, v1.16b | |
| 1663 | + mov v0.16b, v1.16b | |
| 1664 | + udot v17.4s, v30.16b, v30.16b | |
| 1665 | + cbnz w4, 2b | |
| 1666 | + | |
| 1667 | +3: | |
| 1668 | + uaddlv d17, v17.4s | |
| 1669 | + fmov w0, s17 | |
| 1670 | + | |
| 1671 | + ret | |
| 1672 | +endfunc | |
| 1673 | + | |
| 1674 | +DISABLE_DOTPROD | |
| 1675 | +#endif | |