checkasm --bench for Raspberry Pi 5 Model B Rev 1.0: sse_0_c: 241.5 sse_0_neon: 37.2 sse_0_dotprod: 22.2 vsse_4_c: 148.7 vsse_4_neon: 31.0 vsse_4_dotprod: 15.7
Unified · Split
| @@ -77,6 +77,13 @@ int vsse8_neon(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2, · 76 unmodified lines | |||
| 77 | int vsse_intra8_neon(MpegEncContext *c, const uint8_t *s, const uint8_t *dummy, | 77 | int vsse_intra8_neon(MpegEncContext *c, const uint8_t *s, const uint8_t *dummy, |
| 78 | ptrdiff_t stride, int h); | 78 | ptrdiff_t stride, int h); |
| 79 | 79 | ||
| 80 | #if HAVE_DOTPROD | ||
| 81 | int sse16_neon_dotprod(MpegEncContext *v, const uint8_t *pix1, const uint8_t *pix2, | ||
| 82 | ptrdiff_t stride, int h); | ||
| 83 | int vsse_intra16_neon_dotprod(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2, | ||
| 84 | ptrdiff_t stride, int h); | ||
| 85 | #endif | ||
| 86 | |||
| 80 | av_cold void ff_me_cmp_init_aarch64(MECmpContext *c, AVCodecContext *avctx) | 87 | av_cold void ff_me_cmp_init_aarch64(MECmpContext *c, AVCodecContext *avctx) |
| 81 | { | 88 | { |
| 82 | int cpu_flags = av_get_cpu_flags(); | 89 | int cpu_flags = av_get_cpu_flags(); |
| @@ -113,6 +120,13 @@ av_cold void ff_me_cmp_init_aarch64(MECmpContext *c, AVCodecContext *avctx) · 30 unmodified lines | |||
| 113 | c->median_sad[0] = pix_median_abs16_neon; | 120 | c->median_sad[0] = pix_median_abs16_neon; |
| 114 | c->median_sad[1] = pix_median_abs8_neon; | 121 | c->median_sad[1] = pix_median_abs8_neon; |
| 115 | } | 122 | } |
| 123 | |||
| 124 | #if HAVE_DOTPROD | ||
| 125 | if (have_dotprod(cpu_flags)) { | ||
| 126 | c->sse[0] = sse16_neon_dotprod; | ||
| 127 | c->vsse[4] = vsse_intra16_neon_dotprod; | ||
| 128 | } | ||
| 129 | #endif | ||
| 116 | } | 130 | } |
| 117 | 131 | ||
| 118 | int nsse16_neon_wrapper(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2, | 132 | int nsse16_neon_wrapper(MpegEncContext *c, const uint8_t *s1, const uint8_t *s2, |
| @@ -1559,3 +1559,117 @@ function pix_median_abs8_neon, export=1 · 1558 unmodified lines | |||
| 1559 | ret | 1559 | ret |
| 1560 | 1560 | ||
| 1561 | endfunc | 1561 | endfunc |
| 1562 | |||
| 1563 | #if HAVE_DOTPROD | ||
| 1564 | ENABLE_DOTPROD | ||
| 1565 | |||
| 1566 | function sse16_neon_dotprod, export=1 | ||
| 1567 | // x0 - unused | ||
| 1568 | // x1 - pix1 | ||
| 1569 | // x2 - pix2 | ||
| 1570 | // x3 - stride | ||
| 1571 | // w4 - h | ||
| 1572 | |||
| 1573 | cmp w4, #4 | ||
| 1574 | movi v17.4s, #0 | ||
| 1575 | b.lt 2f | ||
| 1576 | |||
| 1577 | // Make 4 iterations at once | ||
| 1578 | 1: | ||
| 1579 | |||
| 1580 | // res = abs(pix1[0] - pix2[0]) | ||
| 1581 | // res * res | ||
| 1582 | |||
| 1583 | ld1 {v0.16b}, [x1], x3 // Load pix1 vector for first iteration | ||
| 1584 | ld1 {v1.16b}, [x2], x3 // Load pix2 vector for first iteration | ||
| 1585 | ld1 {v2.16b}, [x1], x3 // Load pix1 vector for second iteration | ||
| 1586 | uabd v30.16b, v0.16b, v1.16b // Absolute difference, first iteration | ||
| 1587 | ld1 {v3.16b}, [x2], x3 // Load pix2 vector for second iteration | ||
| 1588 | udot v17.4s, v30.16b, v30.16b | ||
| 1589 | uabd v27.16b, v2.16b, v3.16b // Absolute difference, second iteration | ||
| 1590 | ld1 {v4.16b}, [x1], x3 // Load pix1 for third iteration | ||
| 1591 | udot v17.4s, v27.16b, v27.16b | ||
| 1592 | ld1 {v5.16b}, [x2], x3 // Load pix2 for third iteration | ||
| 1593 | uabd v24.16b, v4.16b, v5.16b // Absolute difference, third iteration | ||
| 1594 | ld1 {v6.16b}, [x1], x3 // Load pix1 for fourth iteration | ||
| 1595 | udot v17.4s, v24.16b, v24.16b | ||
| 1596 | ld1 {v7.16b}, [x2], x3 // Load pix2 for fouth iteration | ||
| 1597 | uabd v21.16b, v6.16b, v7.16b // Absolute difference, fourth iteration | ||
| 1598 | sub w4, w4, #4 // h -= 4 | ||
| 1599 | udot v17.4s, v21.16b, v21.16b | ||
| 1600 | cmp w4, #4 | ||
| 1601 | b.ge 1b | ||
| 1602 | |||
| 1603 | cbz w4, 3f | ||
| 1604 | |||
| 1605 | // iterate by one | ||
| 1606 | 2: | ||
| 1607 | |||
| 1608 | ld1 {v0.16b}, [x1], x3 // Load pix1 | ||
| 1609 | ld1 {v1.16b}, [x2], x3 // Load pix2 | ||
| 1610 | |||
| 1611 | uabd v30.16b, v0.16b, v1.16b | ||
| 1612 | subs w4, w4, #1 | ||
| 1613 | udot v17.4s, v30.16b, v30.16b | ||
| 1614 | |||
| 1615 | b.ne 2b | ||
| 1616 | |||
| 1617 | 3: | ||
| 1618 | uaddlv d16, v17.4s // add up accumulator vector | ||
| 1619 | |||
| 1620 | fmov w0, s16 | ||
| 1621 | |||
| 1622 | ret | ||
| 1623 | endfunc | ||
| 1624 | |||
| 1625 | function vsse_intra16_neon_dotprod, export=1 | ||
| 1626 | // x0 unused | ||
| 1627 | // x1 uint8_t *pix1 | ||
| 1628 | // x2 uint8_t *dummy | ||
| 1629 | // x3 ptrdiff_t stride | ||
| 1630 | // w4 int h | ||
| 1631 | |||
| 1632 | ld1 {v0.16b}, [x1], x3 | ||
| 1633 | movi v17.4s, #0 | ||
| 1634 | |||
| 1635 | sub w4, w4, #1 // we need to make h-1 iterations | ||
| 1636 | cmp w4, #3 | ||
| 1637 | b.lt 2f | ||
| 1638 | |||
| 1639 | 1: | ||
| 1640 | // v = abs( pix1[0] - pix1[0 + stride] ) | ||
| 1641 | // score = sum( v * v ) | ||
| 1642 | ld1 {v1.16b}, [x1], x3 | ||
| 1643 | ld1 {v2.16b}, [x1], x3 | ||
| 1644 | uabd v30.16b, v0.16b, v1.16b | ||
| 1645 | ld1 {v3.16b}, [x1], x3 | ||
| 1646 | udot v17.4s, v30.16b, v30.16b | ||
| 1647 | uabd v27.16b, v1.16b, v2.16b | ||
| 1648 | udot v17.4s, v27.16b, v27.16b | ||
| 1649 | uabd v25.16b, v2.16b, v3.16b | ||
| 1650 | sub w4, w4, #3 | ||
| 1651 | udot v17.4s, v25.16b, v25.16b | ||
| 1652 | cmp w4, #3 | ||
| 1653 | mov v0.16b, v3.16b | ||
| 1654 | |||
| 1655 | b.ge 1b | ||
| 1656 | cbz w4, 3f | ||
| 1657 | |||
| 1658 | // iterate by one | ||
| 1659 | 2: | ||
| 1660 | ld1 {v1.16b}, [x1], x3 | ||
| 1661 | subs w4, w4, #1 | ||
| 1662 | uabd v30.16b, v0.16b, v1.16b | ||
| 1663 | mov v0.16b, v1.16b | ||
| 1664 | udot v17.4s, v30.16b, v30.16b | ||
| 1665 | cbnz w4, 2b | ||
| 1666 | |||
| 1667 | 3: | ||
| 1668 | uaddlv d17, v17.4s | ||
| 1669 | fmov w0, s17 | ||
| 1670 | |||
| 1671 | ret | ||
| 1672 | endfunc | ||
| 1673 | |||
| 1674 | DISABLE_DOTPROD | ||
| 1675 | #endif | ||