History

aarch64: hevc: Implement a neon version of hevc_epel_uni_w_h*_8

AWS Graviton 3: put_hevc_epel_uni_w_h4_8_c: 97.2 put_hevc_epel_uni_w_h4_8_neon: 41.2 put_hevc_epel_uni_w_h4_8_i8mm: 35.2 put_hevc_epel_uni_w_h6_8_c: 203.7 put_hevc_epel_uni_w_h6_8_neon: 84.7 put_hevc_epel_uni_w_h6_8_i8mm: 74.7 put_hevc_epel_uni_w_h8_8_c: 345.7 put_hevc_epel_uni_w_h8_8_neon: 94.0 put_hevc_epel_uni_w_h8_8_i8mm: 80.7 put_hevc_epel_uni_w_h12_8_c: 768.7 put_hevc_epel_uni_w_h12_8_neon: 196.7 put_hevc_epel_uni_w_h12_8_i8mm: 169.7 put_hevc_epel_uni_w_h16_8_c: 1313.0 put_hevc_epel_uni_w_h16_8_neon: 290.7 put_hevc_epel_uni_w_h16_8_i8mm: 238.0 put_hevc_epel_uni_w_h24_8_c: 2877.5 put_hevc_epel_uni_w_h24_8_neon: 650.0 put_hevc_epel_uni_w_h24_8_i8mm: 512.0 put_hevc_epel_uni_w_h32_8_c: 5113.5 put_hevc_epel_uni_w_h32_8_neon: 1129.5 put_hevc_epel_uni_w_h32_8_i8mm: 739.2 put_hevc_epel_uni_w_h48_8_c: 11757.0 put_hevc_epel_uni_w_h48_8_neon: 2518.7 put_hevc_epel_uni_w_h48_8_i8mm: 1688.5 put_hevc_epel_uni_w_h64_8_c: 20478.0 put_hevc_epel_uni_w_h64_8_neon: 4411.7 put_hevc_epel_uni_w_h64_8_i8mm: 2884.0

Martin Storsjö committed 2 years ago

Commit 54af5551 parent 6d384292 files · +319 −13Browse files

Unified · Split

2 files
  1. libavcodec/aarch64/hevcdsp_epel_neon.S +313 −13
  2. libavcodec/aarch64/hevcdsp_init_aarch64.c +6 −0
libavcodec/aarch64/hevcdsp_epel_neon.S+313−13
@@ -1520,6 +1520,319 @@ function ff_hevc_put_hevc_epel_h32_8_neon, export=1 · 1519 unmodified lines
15201520 ret
15211521 endfunc
15221522
1523+.macro EPEL_UNI_W_H_HEADER elems=4s
1524+ ldr x12, [sp]
1525+ sub x2, x2, #1
1526+ movrel x9, epel_filters
1527+ add x9, x9, x12, lsl #2
1528+ ld1r {v28.4s}, [x9]
1529+ mov w10, #-6
1530+ sub w10, w10, w5
1531+ dup v30.\elems, w6
1532+ dup v31.4s, w10
1533+ dup v29.4s, w7
1534+.endm
1535+
1536+function ff_hevc_put_hevc_epel_uni_w_h4_8_neon, export=1
1537+ EPEL_UNI_W_H_HEADER 4h
1538+ sxtl v0.8h, v28.8b
1539+1:
1540+ ld1 {v4.8b}, [x2], x3
1541+ subs w4, w4, #1
1542+ uxtl v4.8h, v4.8b
1543+ ext v5.16b, v4.16b, v4.16b, #2
1544+ ext v6.16b, v4.16b, v4.16b, #4
1545+ ext v7.16b, v4.16b, v4.16b, #6
1546+ mul v16.4h, v4.4h, v0.h[0]
1547+ mla v16.4h, v5.4h, v0.h[1]
1548+ mla v16.4h, v6.4h, v0.h[2]
1549+ mla v16.4h, v7.4h, v0.h[3]
1550+ smull v16.4s, v16.4h, v30.4h
1551+ sqrshl v16.4s, v16.4s, v31.4s
1552+ sqadd v16.4s, v16.4s, v29.4s
1553+ sqxtn v16.4h, v16.4s
1554+ sqxtun v16.8b, v16.8h
1555+ str s16, [x0]
1556+ add x0, x0, x1
1557+ b.hi 1b
1558+ ret
1559+endfunc
1560+
1561+function ff_hevc_put_hevc_epel_uni_w_h6_8_neon, export=1
1562+ EPEL_UNI_W_H_HEADER 8h
1563+ sub x1, x1, #4
1564+ sxtl v0.8h, v28.8b
1565+1:
1566+ ld1 {v3.8b, v4.8b}, [x2], x3
1567+ subs w4, w4, #1
1568+ uxtl v3.8h, v3.8b
1569+ uxtl v4.8h, v4.8b
1570+ ext v5.16b, v3.16b, v4.16b, #2
1571+ ext v6.16b, v3.16b, v4.16b, #4
1572+ ext v7.16b, v3.16b, v4.16b, #6
1573+ mul v16.8h, v3.8h, v0.h[0]
1574+ mla v16.8h, v5.8h, v0.h[1]
1575+ mla v16.8h, v6.8h, v0.h[2]
1576+ mla v16.8h, v7.8h, v0.h[3]
1577+ smull v17.4s, v16.4h, v30.4h
1578+ smull2 v18.4s, v16.8h, v30.8h
1579+ sqrshl v17.4s, v17.4s, v31.4s
1580+ sqrshl v18.4s, v18.4s, v31.4s
1581+ sqadd v17.4s, v17.4s, v29.4s
1582+ sqadd v18.4s, v18.4s, v29.4s
1583+ sqxtn v16.4h, v17.4s
1584+ sqxtn2 v16.8h, v18.4s
1585+ sqxtun v16.8b, v16.8h
1586+ str s16, [x0], #4
1587+ st1 {v16.h}[2], [x0], x1
1588+ b.hi 1b
1589+ ret
1590+endfunc
1591+
1592+function ff_hevc_put_hevc_epel_uni_w_h8_8_neon, export=1
1593+ EPEL_UNI_W_H_HEADER 8h
1594+ sxtl v0.8h, v28.8b
1595+1:
1596+ ld1 {v3.8b, v4.8b}, [x2], x3
1597+ subs w4, w4, #1
1598+ uxtl v3.8h, v3.8b
1599+ uxtl v4.8h, v4.8b
1600+ ext v5.16b, v3.16b, v4.16b, #2
1601+ ext v6.16b, v3.16b, v4.16b, #4
1602+ ext v7.16b, v3.16b, v4.16b, #6
1603+ mul v16.8h, v3.8h, v0.h[0]
1604+ mla v16.8h, v5.8h, v0.h[1]
1605+ mla v16.8h, v6.8h, v0.h[2]
1606+ mla v16.8h, v7.8h, v0.h[3]
1607+ smull v17.4s, v16.4h, v30.4h
1608+ smull2 v18.4s, v16.8h, v30.8h
1609+ sqrshl v17.4s, v17.4s, v31.4s
1610+ sqrshl v18.4s, v18.4s, v31.4s
1611+ sqadd v17.4s, v17.4s, v29.4s
1612+ sqadd v18.4s, v18.4s, v29.4s
1613+ sqxtn v16.4h, v17.4s
1614+ sqxtn2 v16.8h, v18.4s
1615+ sqxtun v16.8b, v16.8h
1616+ st1 {v16.8b}, [x0], x1
1617+ b.hi 1b
1618+ ret
1619+endfunc
1620+
1621+function ff_hevc_put_hevc_epel_uni_w_h12_8_neon, export=1
1622+ EPEL_UNI_W_H_HEADER 8h
1623+ sxtl v0.8h, v28.8b
1624+1:
1625+ ld1 {v3.8b, v4.8b}, [x2], x3
1626+ subs w4, w4, #1
1627+ uxtl v3.8h, v3.8b
1628+ uxtl v4.8h, v4.8b
1629+ ext v5.16b, v3.16b, v4.16b, #2
1630+ ext v6.16b, v3.16b, v4.16b, #4
1631+ ext v7.16b, v3.16b, v4.16b, #6
1632+ ext v20.16b, v4.16b, v4.16b, #2
1633+ ext v21.16b, v4.16b, v4.16b, #4
1634+ ext v22.16b, v4.16b, v4.16b, #6
1635+ mul v16.8h, v3.8h, v0.h[0]
1636+ mla v16.8h, v5.8h, v0.h[1]
1637+ mla v16.8h, v6.8h, v0.h[2]
1638+ mla v16.8h, v7.8h, v0.h[3]
1639+ mul v17.4h, v4.4h, v0.h[0]
1640+ mla v17.4h, v20.4h, v0.h[1]
1641+ mla v17.4h, v21.4h, v0.h[2]
1642+ mla v17.4h, v22.4h, v0.h[3]
1643+ smull v18.4s, v16.4h, v30.4h
1644+ smull2 v19.4s, v16.8h, v30.8h
1645+ smull v20.4s, v17.4h, v30.4h
1646+ sqrshl v18.4s, v18.4s, v31.4s
1647+ sqrshl v19.4s, v19.4s, v31.4s
1648+ sqrshl v20.4s, v20.4s, v31.4s
1649+ sqadd v18.4s, v18.4s, v29.4s
1650+ sqadd v19.4s, v19.4s, v29.4s
1651+ sqadd v20.4s, v20.4s, v29.4s
1652+ sqxtn v16.4h, v18.4s
1653+ sqxtn2 v16.8h, v19.4s
1654+ sqxtn v17.4h, v20.4s
1655+ sqxtun v16.8b, v16.8h
1656+ sqxtun v17.8b, v17.8h
1657+ str d16, [x0]
1658+ str s17, [x0, #8]
1659+ add x0, x0, x1
1660+ b.hi 1b
1661+ ret
1662+endfunc
1663+
1664+function ff_hevc_put_hevc_epel_uni_w_h16_8_neon, export=1
1665+ EPEL_UNI_W_H_HEADER 8h
1666+ sxtl v0.8h, v28.8b
1667+1:
1668+ ld1 {v1.8b, v2.8b, v3.8b}, [x2], x3
1669+ subs w4, w4, #1
1670+ uxtl v1.8h, v1.8b
1671+ uxtl v2.8h, v2.8b
1672+ uxtl v3.8h, v3.8b
1673+ ext v5.16b, v1.16b, v2.16b, #2
1674+ ext v6.16b, v1.16b, v2.16b, #4
1675+ ext v7.16b, v1.16b, v2.16b, #6
1676+ ext v20.16b, v2.16b, v3.16b, #2
1677+ ext v21.16b, v2.16b, v3.16b, #4
1678+ ext v22.16b, v2.16b, v3.16b, #6
1679+ mul v16.8h, v1.8h, v0.h[0]
1680+ mla v16.8h, v5.8h, v0.h[1]
1681+ mla v16.8h, v6.8h, v0.h[2]
1682+ mla v16.8h, v7.8h, v0.h[3]
1683+ mul v17.8h, v2.8h, v0.h[0]
1684+ mla v17.8h, v20.8h, v0.h[1]
1685+ mla v17.8h, v21.8h, v0.h[2]
1686+ mla v17.8h, v22.8h, v0.h[3]
1687+ smull v18.4s, v16.4h, v30.4h
1688+ smull2 v19.4s, v16.8h, v30.8h
1689+ smull v20.4s, v17.4h, v30.4h
1690+ smull2 v21.4s, v17.8h, v30.8h
1691+ sqrshl v18.4s, v18.4s, v31.4s
1692+ sqrshl v19.4s, v19.4s, v31.4s
1693+ sqrshl v20.4s, v20.4s, v31.4s
1694+ sqrshl v21.4s, v21.4s, v31.4s
1695+ sqadd v18.4s, v18.4s, v29.4s
1696+ sqadd v19.4s, v19.4s, v29.4s
1697+ sqadd v20.4s, v20.4s, v29.4s
1698+ sqadd v21.4s, v21.4s, v29.4s
1699+ sqxtn v16.4h, v18.4s
1700+ sqxtn2 v16.8h, v19.4s
1701+ sqxtn v17.4h, v20.4s
1702+ sqxtn2 v17.8h, v21.4s
1703+ sqxtun v16.8b, v16.8h
1704+ sqxtun v17.8b, v17.8h
1705+ st1 {v16.8b, v17.8b}, [x0], x1
1706+ b.hi 1b
1707+ ret
1708+endfunc
1709+
1710+function ff_hevc_put_hevc_epel_uni_w_h24_8_neon, export=1
1711+ EPEL_UNI_W_H_HEADER 8h
1712+ sxtl v0.8h, v28.8b
1713+1:
1714+ ld1 {v1.8b, v2.8b, v3.8b, v4.8b}, [x2], x3
1715+ subs w4, w4, #1
1716+ uxtl v1.8h, v1.8b
1717+ uxtl v2.8h, v2.8b
1718+ uxtl v3.8h, v3.8b
1719+ uxtl v4.8h, v4.8b
1720+ ext v5.16b, v1.16b, v2.16b, #2
1721+ ext v6.16b, v1.16b, v2.16b, #4
1722+ ext v7.16b, v1.16b, v2.16b, #6
1723+ ext v20.16b, v2.16b, v3.16b, #2
1724+ ext v21.16b, v2.16b, v3.16b, #4
1725+ ext v22.16b, v2.16b, v3.16b, #6
1726+ ext v23.16b, v3.16b, v4.16b, #2
1727+ ext v24.16b, v3.16b, v4.16b, #4
1728+ ext v25.16b, v3.16b, v4.16b, #6
1729+ mul v16.8h, v1.8h, v0.h[0]
1730+ mla v16.8h, v5.8h, v0.h[1]
1731+ mla v16.8h, v6.8h, v0.h[2]
1732+ mla v16.8h, v7.8h, v0.h[3]
1733+ mul v17.8h, v2.8h, v0.h[0]
1734+ mla v17.8h, v20.8h, v0.h[1]
1735+ mla v17.8h, v21.8h, v0.h[2]
1736+ mla v17.8h, v22.8h, v0.h[3]
1737+ mul v18.8h, v3.8h, v0.h[0]
1738+ mla v18.8h, v23.8h, v0.h[1]
1739+ mla v18.8h, v24.8h, v0.h[2]
1740+ mla v18.8h, v25.8h, v0.h[3]
1741+ smull v20.4s, v16.4h, v30.4h
1742+ smull2 v21.4s, v16.8h, v30.8h
1743+ smull v22.4s, v17.4h, v30.4h
1744+ smull2 v23.4s, v17.8h, v30.8h
1745+ smull v24.4s, v18.4h, v30.4h
1746+ smull2 v25.4s, v18.8h, v30.8h
1747+ sqrshl v20.4s, v20.4s, v31.4s
1748+ sqrshl v21.4s, v21.4s, v31.4s
1749+ sqrshl v22.4s, v22.4s, v31.4s
1750+ sqrshl v23.4s, v23.4s, v31.4s
1751+ sqrshl v24.4s, v24.4s, v31.4s
1752+ sqrshl v25.4s, v25.4s, v31.4s
1753+ sqadd v20.4s, v20.4s, v29.4s
1754+ sqadd v21.4s, v21.4s, v29.4s
1755+ sqadd v22.4s, v22.4s, v29.4s
1756+ sqadd v23.4s, v23.4s, v29.4s
1757+ sqadd v24.4s, v24.4s, v29.4s
1758+ sqadd v25.4s, v25.4s, v29.4s
1759+ sqxtn v16.4h, v20.4s
1760+ sqxtn2 v16.8h, v21.4s
1761+ sqxtn v17.4h, v22.4s
1762+ sqxtn2 v17.8h, v23.4s
1763+ sqxtn v18.4h, v24.4s
1764+ sqxtn2 v18.8h, v25.4s
1765+ sqxtun v16.8b, v16.8h
1766+ sqxtun v17.8b, v17.8h
1767+ sqxtun v18.8b, v18.8h
1768+ st1 {v16.8b, v17.8b, v18.8b}, [x0], x1
1769+ b.hi 1b
1770+ ret
1771+endfunc
1772+
1773+function ff_hevc_put_hevc_epel_uni_w_h32_8_neon, export=1
1774+ EPEL_UNI_W_H_HEADER 8h
1775+ ldr w10, [sp, #16] // width
1776+ ld1 {v1.8b}, [x2], #8
1777+ sub x3, x3, w10, uxtw // decrement src stride
1778+ mov w11, w10 // original width
1779+ sub x3, x3, #8 // decrement src stride
1780+ sub x1, x1, w10, uxtw // decrement dst stride
1781+ sxtl v0.8h, v28.8b
1782+ uxtl v1.8h, v1.8b
1783+1:
1784+ ld1 {v2.8b, v3.8b}, [x2], #16
1785+ subs w10, w10, #16 // width
1786+ uxtl v2.8h, v2.8b
1787+ uxtl v3.8h, v3.8b
1788+ ext v5.16b, v1.16b, v2.16b, #2
1789+ ext v6.16b, v1.16b, v2.16b, #4
1790+ ext v7.16b, v1.16b, v2.16b, #6
1791+ ext v20.16b, v2.16b, v3.16b, #2
1792+ ext v21.16b, v2.16b, v3.16b, #4
1793+ ext v22.16b, v2.16b, v3.16b, #6
1794+ mul v16.8h, v1.8h, v0.h[0]
1795+ mla v16.8h, v5.8h, v0.h[1]
1796+ mla v16.8h, v6.8h, v0.h[2]
1797+ mla v16.8h, v7.8h, v0.h[3]
1798+ mul v17.8h, v2.8h, v0.h[0]
1799+ mla v17.8h, v20.8h, v0.h[1]
1800+ mla v17.8h, v21.8h, v0.h[2]
1801+ mla v17.8h, v22.8h, v0.h[3]
1802+ smull v18.4s, v16.4h, v30.4h
1803+ smull2 v19.4s, v16.8h, v30.8h
1804+ smull v20.4s, v17.4h, v30.4h
1805+ smull2 v21.4s, v17.8h, v30.8h
1806+ sqrshl v18.4s, v18.4s, v31.4s
1807+ sqrshl v19.4s, v19.4s, v31.4s
1808+ sqrshl v20.4s, v20.4s, v31.4s
1809+ sqrshl v21.4s, v21.4s, v31.4s
1810+ sqadd v18.4s, v18.4s, v29.4s
1811+ sqadd v19.4s, v19.4s, v29.4s
1812+ sqadd v20.4s, v20.4s, v29.4s
1813+ sqadd v21.4s, v21.4s, v29.4s
1814+ sqxtn v16.4h, v18.4s
1815+ sqxtn2 v16.8h, v19.4s
1816+ sqxtn v17.4h, v20.4s
1817+ sqxtn2 v17.8h, v21.4s
1818+ sqxtun v16.8b, v16.8h
1819+ sqxtun v17.8b, v17.8h
1820+ st1 {v16.8b, v17.8b}, [x0], #16
1821+ mov v1.16b, v3.16b
1822+ b.gt 1b
1823+ subs w4, w4, #1 // height
1824+ add x2, x2, x3
1825+ b.le 9f
1826+ ld1 {v1.8b}, [x2], #8
1827+ mov w10, w11
1828+ add x0, x0, x1
1829+ uxtl v1.8h, v1.8b
1830+ b 1b
1831+9:
1832+ ret
1833+endfunc
1834+
1835+
15231836 #if HAVE_I8MM
15241837 ENABLE_I8MM
15251838 function ff_hevc_put_hevc_epel_h4_8_neon_i8mm, export=1
@@ -2410,19 +2723,6 @@ function ff_hevc_put_hevc_epel_uni_hv64_8_neon_i8mm, export=1 · 884 unmodified lines
24102723 ret
24112724 endfunc
24122725
2413−.macro EPEL_UNI_W_H_HEADER
2414− ldr x12, [sp]
2415− sub x2, x2, #1
2416− movrel x9, epel_filters
2417− add x9, x9, x12, lsl #2
2418− ld1r {v28.4s}, [x9]
2419− mov w10, #-6
2420− sub w10, w10, w5
2421− dup v30.4s, w6
2422− dup v31.4s, w10
2423− dup v29.4s, w7
2424−.endm
2425−
24262726
24272727 function ff_hevc_put_hevc_epel_uni_w_h4_8_neon_i8mm, export=1
24282728 EPEL_UNI_W_H_HEADER
libavcodec/aarch64/hevcdsp_init_aarch64.c+6−0
@@ -235,6 +235,11 @@ NEON8_FNPROTO(epel_hv, (int16_t *dst, · 234 unmodified lines
235235 const uint8_t *src, ptrdiff_t srcstride,
236236 int height, intptr_t mx, intptr_t my, int width), _i8mm);
237237
238+NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
239+ const uint8_t *_src, ptrdiff_t _srcstride,
240+ int height, int denom, int wx, int ox,
241+ intptr_t mx, intptr_t my, int width),);
242+
238243 NEON8_FNPROTO(epel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride,
239244 const uint8_t *_src, ptrdiff_t _srcstride,
240245 int height, int denom, int wx, int ox,
@@ -400,6 +405,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 159 unmodified lines
400405 NEON8_FNASSIGN_PARTIAL_4(c->put_hevc_qpel_uni_w, 1, 0, qpel_uni_w_v,);
401406
402407 NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel, 0, 1, epel_h,);
408+ NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h,);
403409
404410 if (have_i8mm(cpu_flags)) {
405411 NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t