History

aarch64: hevc: Implement a neon version of hevc_epel_uni_w_h*_8

AWS Graviton 3: put_hevc_epel_uni_w_h4_8_c: 97.2 put_hevc_epel_uni_w_h4_8_neon: 41.2 put_hevc_epel_uni_w_h4_8_i8mm: 35.2 put_hevc_epel_uni_w_h6_8_c: 203.7 put_hevc_epel_uni_w_h6_8_neon: 84.7 put_hevc_epel_uni_w_h6_8_i8mm: 74.7 put_hevc_epel_uni_w_h8_8_c: 345.7 put_hevc_epel_uni_w_h8_8_neon: 94.0 put_hevc_epel_uni_w_h8_8_i8mm: 80.7 put_hevc_epel_uni_w_h12_8_c: 768.7 put_hevc_epel_uni_w_h12_8_neon: 196.7 put_hevc_epel_uni_w_h12_8_i8mm: 169.7 put_hevc_epel_uni_w_h16_8_c: 1313.0 put_hevc_epel_uni_w_h16_8_neon: 290.7 put_hevc_epel_uni_w_h16_8_i8mm: 238.0 put_hevc_epel_uni_w_h24_8_c: 2877.5 put_hevc_epel_uni_w_h24_8_neon: 650.0 put_hevc_epel_uni_w_h24_8_i8mm: 512.0 put_hevc_epel_uni_w_h32_8_c: 5113.5 put_hevc_epel_uni_w_h32_8_neon: 1129.5 put_hevc_epel_uni_w_h32_8_i8mm: 739.2 put_hevc_epel_uni_w_h48_8_c: 11757.0 put_hevc_epel_uni_w_h48_8_neon: 2518.7 put_hevc_epel_uni_w_h48_8_i8mm: 1688.5 put_hevc_epel_uni_w_h64_8_c: 20478.0 put_hevc_epel_uni_w_h64_8_neon: 4411.7 put_hevc_epel_uni_w_h64_8_i8mm: 2884.0

Martin Storsjö committed 2 years ago

Commit 54af5551 parent 6d384292 files · +319 −13Browse files

Unified · Split

2 files
  1. libavcodec/aarch64/hevcdsp_epel_neon.S +313 −13
  2. libavcodec/aarch64/hevcdsp_init_aarch64.c +6 −0
libavcodec/aarch64/hevcdsp_epel_neon.S+313−13
@@ -1520,6 +1520,319 @@ function ff_hevc_put_hevc_epel_h32_8_neon, export=1 · 1519 unmodified lines
1520 ret1520 ret
1521endfunc1521endfunc
1522 1522
1523.macro EPEL_UNI_W_H_HEADER elems=4s
1524 ldr x12, [sp]
1525 sub x2, x2, #1
1526 movrel x9, epel_filters
1527 add x9, x9, x12, lsl #2
1528 ld1r {v28.4s}, [x9]
1529 mov w10, #-6
1530 sub w10, w10, w5
1531 dup v30.\elems, w6
1532 dup v31.4s, w10
1533 dup v29.4s, w7
1534.endm
1535
1536function ff_hevc_put_hevc_epel_uni_w_h4_8_neon, export=1
1537 EPEL_UNI_W_H_HEADER 4h
1538 sxtl v0.8h, v28.8b
15391:
1540 ld1 {v4.8b}, [x2], x3
1541 subs w4, w4, #1
1542 uxtl v4.8h, v4.8b
1543 ext v5.16b, v4.16b, v4.16b, #2
1544 ext v6.16b, v4.16b, v4.16b, #4
1545 ext v7.16b, v4.16b, v4.16b, #6
1546 mul v16.4h, v4.4h, v0.h[0]
1547 mla v16.4h, v5.4h, v0.h[1]
1548 mla v16.4h, v6.4h, v0.h[2]
1549 mla v16.4h, v7.4h, v0.h[3]
1550 smull v16.4s, v16.4h, v30.4h
1551 sqrshl v16.4s, v16.4s, v31.4s
1552 sqadd v16.4s, v16.4s, v29.4s
1553 sqxtn v16.4h, v16.4s
1554 sqxtun v16.8b, v16.8h
1555 str s16, [x0]
1556 add x0, x0, x1
1557 b.hi 1b
1558 ret
1559endfunc
1560
1561function ff_hevc_put_hevc_epel_uni_w_h6_8_neon, export=1
1562 EPEL_UNI_W_H_HEADER 8h
1563 sub x1, x1, #4
1564 sxtl v0.8h, v28.8b
15651:
1566 ld1 {v3.8b, v4.8b}, [x2], x3
1567 subs w4, w4, #1
1568 uxtl v3.8h, v3.8b
1569 uxtl v4.8h, v4.8b
1570 ext v5.16b, v3.16b, v4.16b, #2
1571 ext v6.16b, v3.16b, v4.16b, #4
1572 ext v7.16b, v3.16b, v4.16b, #6
1573 mul v16.8h, v3.8h, v0.h[0]
1574 mla v16.8h, v5.8h, v0.h[1]
1575 mla v16.8h, v6.8h, v0.h[2]
1576 mla v16.8h, v7.8h, v0.h[3]
1577 smull v17.4s, v16.4h, v30.4h
1578 smull2 v18.4s, v16.8h, v30.8h
1579 sqrshl v17.4s, v17.4s, v31.4s
1580 sqrshl v18.4s, v18.4s, v31.4s
1581 sqadd v17.4s, v17.4s, v29.4s
1582 sqadd v18.4s, v18.4s, v29.4s
1583 sqxtn v16.4h, v17.4s
1584 sqxtn2 v16.8h, v18.4s
1585 sqxtun v16.8b, v16.8h
1586 str s16, [x0], #4
1587 st1 {v16.h}[2], [x0], x1
1588 b.hi 1b
1589 ret
1590endfunc
1591
1592function ff_hevc_put_hevc_epel_uni_w_h8_8_neon, export=1
1593 EPEL_UNI_W_H_HEADER 8h
1594 sxtl v0.8h, v28.8b
15951:
1596 ld1 {v3.8b, v4.8b}, [x2], x3
1597 subs w4, w4, #1
1598 uxtl v3.8h, v3.8b
1599 uxtl v4.8h, v4.8b
1600 ext v5.16b, v3.16b, v4.16b, #2
1601 ext v6.16b, v3.16b, v4.16b, #4
1602 ext v7.16b, v3.16b, v4.16b, #6
1603 mul v16.8h, v3.8h, v0.h[0]
1604 mla v16.8h, v5.8h, v0.h[1]
1605 mla v16.8h, v6.8h, v0.h[2]
1606 mla v16.8h, v7.8h, v0.h[3]
1607 smull v17.4s, v16.4h, v30.4h
1608 smull2 v18.4s, v16.8h, v30.8h
1609 sqrshl v17.4s, v17.4s, v31.4s
1610 sqrshl v18.4s, v18.4s, v31.4s
1611 sqadd v17.4s, v17.4s, v29.4s
1612 sqadd v18.4s, v18.4s, v29.4s
1613 sqxtn v16.4h, v17.4s
1614 sqxtn2 v16.8h, v18.4s
1615 sqxtun v16.8b, v16.8h
1616 st1 {v16.8b}, [x0], x1
1617 b.hi 1b
1618 ret
1619endfunc
1620
1621function ff_hevc_put_hevc_epel_uni_w_h12_8_neon, export=1
1622 EPEL_UNI_W_H_HEADER 8h
1623 sxtl v0.8h, v28.8b
16241:
1625 ld1 {v3.8b, v4.8b}, [x2], x3
1626 subs w4, w4, #1
1627 uxtl v3.8h, v3.8b
1628 uxtl v4.8h, v4.8b
1629 ext v5.16b, v3.16b, v4.16b, #2
1630 ext v6.16b, v3.16b, v4.16b, #4
1631 ext v7.16b, v3.16b, v4.16b, #6
1632 ext v20.16b, v4.16b, v4.16b, #2
1633 ext v21.16b, v4.16b, v4.16b, #4
1634 ext v22.16b, v4.16b, v4.16b, #6
1635 mul v16.8h, v3.8h, v0.h[0]
1636 mla v16.8h, v5.8h, v0.h[1]
1637 mla v16.8h, v6.8h, v0.h[2]
1638 mla v16.8h, v7.8h, v0.h[3]
1639 mul v17.4h, v4.4h, v0.h[0]
1640 mla v17.4h, v20.4h, v0.h[1]
1641 mla v17.4h, v21.4h, v0.h[2]
1642 mla v17.4h, v22.4h, v0.h[3]
1643 smull v18.4s, v16.4h, v30.4h
1644 smull2 v19.4s, v16.8h, v30.8h
1645 smull v20.4s, v17.4h, v30.4h
1646 sqrshl v18.4s, v18.4s, v31.4s
1647 sqrshl v19.4s, v19.4s, v31.4s
1648 sqrshl v20.4s, v20.4s, v31.4s
1649 sqadd v18.4s, v18.4s, v29.4s
1650 sqadd v19.4s, v19.4s, v29.4s
1651 sqadd v20.4s, v20.4s, v29.4s
1652 sqxtn v16.4h, v18.4s
1653 sqxtn2 v16.8h, v19.4s
1654 sqxtn v17.4h, v20.4s
1655 sqxtun v16.8b, v16.8h
1656 sqxtun v17.8b, v17.8h
1657 str d16, [x0]
1658 str s17, [x0, #8]
1659 add x0, x0, x1
1660 b.hi 1b
1661 ret
1662endfunc
1663
1664function ff_hevc_put_hevc_epel_uni_w_h16_8_neon, export=1
1665 EPEL_UNI_W_H_HEADER 8h
1666 sxtl v0.8h, v28.8b
16671:
1668 ld1 {v1.8b, v2.8b, v3.8b}, [x2], x3
1669 subs w4, w4, #1
1670 uxtl v1.8h, v1.8b
1671 uxtl v2.8h, v2.8b
1672 uxtl v3.8h, v3.8b
1673 ext v5.16b, v1.16b, v2.16b, #2
1674 ext v6.16b, v1.16b, v2.16b, #4
1675 ext v7.16b, v1.16b, v2.16b, #6
1676 ext v20.16b, v2.16b, v3.16b, #2
1677 ext v21.16b, v2.16b, v3.16b, #4
1678 ext v22.16b, v2.16b, v3.16b, #6
1679 mul v16.8h, v1.8h, v0.h[0]
1680 mla v16.8h, v5.8h, v0.h[1]
1681 mla v16.8h, v6.8h, v0.h[2]
1682 mla v16.8h, v7.8h, v0.h[3]
1683 mul v17.8h, v2.8h, v0.h[0]
1684 mla v17.8h, v20.8h, v0.h[1]
1685 mla v17.8h, v21.8h, v0.h[2]
1686 mla v17.8h, v22.8h, v0.h[3]
1687 smull v18.4s, v16.4h, v30.4h
1688 smull2 v19.4s, v16.8h, v30.8h
1689 smull v20.4s, v17.4h, v30.4h
1690 smull2 v21.4s, v17.8h, v30.8h
1691 sqrshl v18.4s, v18.4s, v31.4s
1692 sqrshl v19.4s, v19.4s, v31.4s
1693 sqrshl v20.4s, v20.4s, v31.4s
1694 sqrshl v21.4s, v21.4s, v31.4s
1695 sqadd v18.4s, v18.4s, v29.4s
1696 sqadd v19.4s, v19.4s, v29.4s
1697 sqadd v20.4s, v20.4s, v29.4s
1698 sqadd v21.4s, v21.4s, v29.4s
1699 sqxtn v16.4h, v18.4s
1700 sqxtn2 v16.8h, v19.4s
1701 sqxtn v17.4h, v20.4s
1702 sqxtn2 v17.8h, v21.4s
1703 sqxtun v16.8b, v16.8h
1704 sqxtun v17.8b, v17.8h
1705 st1 {v16.8b, v17.8b}, [x0], x1
1706 b.hi 1b
1707 ret
1708endfunc
1709
1710function ff_hevc_put_hevc_epel_uni_w_h24_8_neon, export=1
1711 EPEL_UNI_W_H_HEADER 8h
1712 sxtl v0.8h, v28.8b
17131:
1714 ld1 {v1.8b, v2.8b, v3.8b, v4.8b}, [x2], x3
1715 subs w4, w4, #1
1716 uxtl v1.8h, v1.8b
1717 uxtl v2.8h, v2.8b
1718 uxtl v3.8h, v3.8b
1719 uxtl v4.8h, v4.8b
1720 ext v5.16b, v1.16b, v2.16b, #2
1721 ext v6.16b, v1.16b, v2.16b, #4
1722 ext v7.16b, v1.16b, v2.16b, #6
1723 ext v20.16b, v2.16b, v3.16b, #2
1724 ext v21.16b, v2.16b, v3.16b, #4
1725 ext v22.16b, v2.16b, v3.16b, #6
1726 ext v23.16b, v3.16b, v4.16b, #2
1727 ext v24.16b, v3.16b, v4.16b, #4
1728 ext v25.16b, v3.16b, v4.16b, #6
1729 mul v16.8h, v1.8h, v0.h[0]
1730 mla v16.8h, v5.8h, v0.h[1]
1731 mla v16.8h, v6.8h, v0.h[2]
1732 mla v16.8h, v7.8h, v0.h[3]
1733 mul v17.8h, v2.8h, v0.h[0]
1734 mla v17.8h, v20.8h, v0.h[1]
1735 mla v17.8h, v21.8h, v0.h[2]
1736 mla v17.8h, v22.8h, v0.h[3]
1737 mul v18.8h, v3.8h, v0.h[0]
1738 mla v18.8h, v23.8h, v0.h[1]
1739 mla v18.8h, v24.8h, v0.h[2]
1740 mla v18.8h, v25.8h, v0.h[3]
1741 smull v20.4s, v16.4h, v30.4h
1742 smull2 v21.4s, v16.8h, v30.8h
1743 smull v22.4s, v17.4h, v30.4h
1744 smull2 v23.4s, v17.8h, v30.8h
1745 smull v24.4s, v18.4h, v30.4h
1746 smull2 v25.4s, v18.8h, v30.8h
1747 sqrshl v20.4s, v20.4s, v31.4s
1748 sqrshl v21.4s, v21.4s, v31.4s
1749 sqrshl v22.4s, v22.4s, v31.4s
1750 sqrshl v23.4s, v23.4s, v31.4s
1751 sqrshl v24.4s, v24.4s, v31.4s
1752 sqrshl v25.4s, v25.4s, v31.4s
1753 sqadd v20.4s, v20.4s, v29.4s
1754 sqadd v21.4s, v21.4s, v29.4s
1755 sqadd v22.4s, v22.4s, v29.4s
1756 sqadd v23.4s, v23.4s, v29.4s
1757 sqadd v24.4s, v24.4s, v29.4s
1758 sqadd v25.4s, v25.4s, v29.4s
1759 sqxtn v16.4h, v20.4s
1760 sqxtn2 v16.8h, v21.4s
1761 sqxtn v17.4h, v22.4s
1762 sqxtn2 v17.8h, v23.4s
1763 sqxtn v18.4h, v24.4s
1764 sqxtn2 v18.8h, v25.4s
1765 sqxtun v16.8b, v16.8h
1766 sqxtun v17.8b, v17.8h
1767 sqxtun v18.8b, v18.8h
1768 st1 {v16.8b, v17.8b, v18.8b}, [x0], x1
1769 b.hi 1b
1770 ret
1771endfunc
1772
1773function ff_hevc_put_hevc_epel_uni_w_h32_8_neon, export=1
1774 EPEL_UNI_W_H_HEADER 8h
1775 ldr w10, [sp, #16] // width
1776 ld1 {v1.8b}, [x2], #8
1777 sub x3, x3, w10, uxtw // decrement src stride
1778 mov w11, w10 // original width
1779 sub x3, x3, #8 // decrement src stride
1780 sub x1, x1, w10, uxtw // decrement dst stride
1781 sxtl v0.8h, v28.8b
1782 uxtl v1.8h, v1.8b
17831:
1784 ld1 {v2.8b, v3.8b}, [x2], #16
1785 subs w10, w10, #16 // width
1786 uxtl v2.8h, v2.8b
1787 uxtl v3.8h, v3.8b
1788 ext v5.16b, v1.16b, v2.16b, #2
1789 ext v6.16b, v1.16b, v2.16b, #4
1790 ext v7.16b, v1.16b, v2.16b, #6
1791 ext v20.16b, v2.16b, v3.16b, #2
1792 ext v21.16b, v2.16b, v3.16b, #4
1793 ext v22.16b, v2.16b, v3.16b, #6
1794 mul v16.8h, v1.8h, v0.h[0]
1795 mla v16.8h, v5.8h, v0.h[1]
1796 mla v16.8h, v6.8h, v0.h[2]
1797 mla v16.8h, v7.8h, v0.h[3]
1798 mul v17.8h, v2.8h, v0.h[0]
1799 mla v17.8h, v20.8h, v0.h[1]
1800 mla v17.8h, v21.8h, v0.h[2]
1801 mla v17.8h, v22.8h, v0.h[3]
1802 smull v18.4s, v16.4h, v30.4h
1803 smull2 v19.4s, v16.8h, v30.8h
1804 smull v20.4s, v17.4h, v30.4h
1805 smull2 v21.4s, v17.8h, v30.8h
1806 sqrshl v18.4s, v18.4s, v31.4s
1807 sqrshl v19.4s, v19.4s, v31.4s
1808 sqrshl v20.4s, v20.4s, v31.4s
1809 sqrshl v21.4s, v21.4s, v31.4s
1810 sqadd v18.4s, v18.4s, v29.4s
1811 sqadd v19.4s, v19.4s, v29.4s
1812 sqadd v20.4s, v20.4s, v29.4s
1813 sqadd v21.4s, v21.4s, v29.4s
1814 sqxtn v16.4h, v18.4s
1815 sqxtn2 v16.8h, v19.4s
1816 sqxtn v17.4h, v20.4s
1817 sqxtn2 v17.8h, v21.4s
1818 sqxtun v16.8b, v16.8h
1819 sqxtun v17.8b, v17.8h
1820 st1 {v16.8b, v17.8b}, [x0], #16
1821 mov v1.16b, v3.16b
1822 b.gt 1b
1823 subs w4, w4, #1 // height
1824 add x2, x2, x3
1825 b.le 9f
1826 ld1 {v1.8b}, [x2], #8
1827 mov w10, w11
1828 add x0, x0, x1
1829 uxtl v1.8h, v1.8b
1830 b 1b
18319:
1832 ret
1833endfunc
1834
1835
1523#if HAVE_I8MM1836#if HAVE_I8MM
1524ENABLE_I8MM1837ENABLE_I8MM
1525function ff_hevc_put_hevc_epel_h4_8_neon_i8mm, export=11838function ff_hevc_put_hevc_epel_h4_8_neon_i8mm, export=1
@@ -2410,19 +2723,6 @@ function ff_hevc_put_hevc_epel_uni_hv64_8_neon_i8mm, export=1 · 884 unmodified lines
2410 ret2723 ret
2411endfunc2724endfunc
2412 2725
2413.macro EPEL_UNI_W_H_HEADER
2414 ldr x12, [sp]
2415 sub x2, x2, #1
2416 movrel x9, epel_filters
2417 add x9, x9, x12, lsl #2
2418 ld1r {v28.4s}, [x9]
2419 mov w10, #-6
2420 sub w10, w10, w5
2421 dup v30.4s, w6
2422 dup v31.4s, w10
2423 dup v29.4s, w7
2424.endm
2425
2426 2726
2427function ff_hevc_put_hevc_epel_uni_w_h4_8_neon_i8mm, export=12727function ff_hevc_put_hevc_epel_uni_w_h4_8_neon_i8mm, export=1
2428 EPEL_UNI_W_H_HEADER2728 EPEL_UNI_W_H_HEADER
libavcodec/aarch64/hevcdsp_init_aarch64.c+6−0
@@ -235,6 +235,11 @@ NEON8_FNPROTO(epel_hv, (int16_t *dst, · 234 unmodified lines
235 const uint8_t *src, ptrdiff_t srcstride,235 const uint8_t *src, ptrdiff_t srcstride,
236 int height, intptr_t mx, intptr_t my, int width), _i8mm);236 int height, intptr_t mx, intptr_t my, int width), _i8mm);
237 237
238NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
239 const uint8_t *_src, ptrdiff_t _srcstride,
240 int height, int denom, int wx, int ox,
241 intptr_t mx, intptr_t my, int width),);
242
238NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,243NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
239 const uint8_t *_src, ptrdiff_t _srcstride,244 const uint8_t *_src, ptrdiff_t _srcstride,
240 int height, int denom, int wx, int ox,245 int height, int denom, int wx, int ox,
@@ -400,6 +405,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 159 unmodified lines
400 NEON8_FNASSIGN_PARTIAL_4(c->put_hevc_qpel_uni_w, 1, 0, qpel_uni_w_v,);405 NEON8_FNASSIGN_PARTIAL_4(c->put_hevc_qpel_uni_w, 1, 0, qpel_uni_w_v,);
401 406
402 NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel, 0, 1, epel_h,);407 NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel, 0, 1, epel_h,);
408 NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h,);
403 409
404 if (have_i8mm(cpu_flags)) {410 if (have_i8mm(cpu_flags)) {
405 NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);411 NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t