/* * Copyright © 2018, VideoLAN and dav1d authors * Copyright © 2018, Janne Grunau * Copyright © 2018, Martin Storsjo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this *    list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, *    this list of conditions and the following disclaimer in the documentation *    and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "src/arm/asm.S"#include "util.S" .macro avg dst, t0, t1, t2, t3        ld1             {\t0\().8h,\t1\().8h},   [x2],  32        ld1             {\t2\().8h,\t3\().8h},   [x3],  32        add             \t0\().8h,   \t0\().8h,   \t2\().8h        add             \t1\().8h,   \t1\().8h,   \t3\().8h        sqrshrun        \dst\().8b,  \t0\().8h,   #5        sqrshrun2       \dst\().16b, \t1\().8h,   #5.endm .macro w_avg dst, t0, t1, t2, t3        ld1             {\t0\().8h,\t1\().8h},   [x2],  32        ld1             {\t2\().8h,\t3\().8h},   [x3],  32        sub             \t0\().8h,   \t2\().8h,   \t0\().8h        sub             \t1\().8h,   \t3\().8h,   \t1\().8h        sqdmulh         \t0\().8h,   \t0\().8h,   v30.8h        sqdmulh         \t1\().8h,   \t1\().8h,   v30.8h        add             \t0\().8h,   \t2\().8h,   \t0\().8h        add             \t1\().8h,   \t3\().8h,   \t1\().8h        sqrshrun        \dst\().8b,  \t0\().8h,   #4        sqrshrun2       \dst\().16b, \t1\().8h,   #4.endm .macro mask dst, t0, t1, t2, t3        ld1             {v30.16b}, [x6],  16        ld1             {\t0\().8h,\t1\().8h},   [x2],  32        mul             v30.16b, v30.16b, v31.16b        ld1             {\t2\().8h,\t3\().8h},   [x3],  32        shll            v28.8h, v30.8b,  #8        shll2           v29.8h, v30.16b, #8        sub             \t0\().8h,   \t2\().8h,   \t0\().8h        sub             \t1\().8h,   \t3\().8h,   \t1\().8h        sqdmulh         \t0\().8h,   \t0\().8h,   v28.8h        sqdmulh         \t1\().8h,   \t1\().8h,   v29.8h        add             \t0\().8h,   \t2\().8h,   \t0\().8h        add             \t1\().8h,   \t3\().8h,   \t1\().8h        sqrshrun        \dst\().8b,  \t0\().8h,   #4        sqrshrun2       \dst\().16b, \t1\().8h,   #4.endm .macro bidir_fn typefunction \type\()_8bpc_neon, export=1        clz             w4,  w4.ifc \type, w_avg        dup             v30.8h, w6        neg             v30.8h, v30.8h        shl             v30.8h, v30.8h, #11.endif.ifc \type, mask        movi            v31.16b, #256-2.endif        adr             x7,  L(\type\()_tbl)        sub             w4,  w4,  #24        ldrh            w4,  [x7, x4, lsl #1]        \type           v4,  v0,  v1,  v2,  v3        sub             x7,  x7,  w4, uxtw        br              x740:        add             x7,  x0,  x1        lsl             x1,  x1,  #14:        cmp             w5,  #4        st1             {v4.s}[0],  [x0], x1        st1             {v4.s}[1],  [x7], x1        st1             {v4.s}[2],  [x0], x1        st1             {v4.s}[3],  [x7], x1        b.eq            0f        \type           v5,  v0,  v1,  v2,  v3        cmp             w5,  #8        st1             {v5.s}[0],  [x0], x1        st1             {v5.s}[1],  [x7], x1        st1             {v5.s}[2],  [x0], x1        st1             {v5.s}[3],  [x7], x1        b.eq            0f        \type           v4,  v0,  v1,  v2,  v3        st1             {v4.s}[0],  [x0], x1        st1             {v4.s}[1],  [x7], x1        \type           v5,  v0,  v1,  v2,  v3        st1             {v4.s}[2],  [x0], x1        st1             {v4.s}[3],  [x7], x1        st1             {v5.s}[0],  [x0], x1        st1             {v5.s}[1],  [x7], x1        st1             {v5.s}[2],  [x0], x1        st1             {v5.s}[3],  [x7], x1        ret80:        add             x7,  x0,  x1        lsl             x1,  x1,  #18:        st1             {v4.d}[0],  [x0], x1        \type           v5,  v0,  v1,  v2,  v3        st1             {v4.d}[1],  [x7], x1        st1             {v5.d}[0],  [x0], x1        subs            w5,  w5,  #4        st1             {v5.d}[1],  [x7], x1        b.le            0f        \type           v4,  v0,  v1,  v2,  v3        b               8b16:        \type           v5,  v0,  v1,  v2,  v3        st1             {v4.16b}, [x0], x1        \type           v6,  v0,  v1,  v2,  v3        st1             {v5.16b}, [x0], x1        \type           v7,  v0,  v1,  v2,  v3        st1             {v6.16b}, [x0], x1        subs            w5,  w5,  #4        st1             {v7.16b}, [x0], x1        b.le            0f        \type           v4,  v0,  v1,  v2,  v3        b               16b320:        add             x7,  x0,  x1        lsl             x1,  x1,  #132:        \type           v5,  v0,  v1,  v2,  v3        \type           v6,  v0,  v1,  v2,  v3        st1             {v4.16b,v5.16b}, [x0], x1        \type           v7,  v0,  v1,  v2,  v3        subs            w5,  w5,  #2        st1             {v6.16b,v7.16b}, [x7], x1        b.le            0f        \type           v4,  v0,  v1,  v2,  v3        b               32b640:        add             x7,  x0,  x1        lsl             x1,  x1,  #164:        \type           v5,  v0,  v1,  v2,  v3        \type           v6,  v0,  v1,  v2,  v3        \type           v7,  v0,  v1,  v2,  v3        \type           v16, v0,  v1,  v2,  v3        \type           v17, v0,  v1,  v2,  v3        st1             {v4.16b,v5.16b,v6.16b,v7.16b}, [x0], x1        \type           v18, v0,  v1,  v2,  v3        \type           v19, v0,  v1,  v2,  v3        subs            w5,  w5,  #2        st1             {v16.16b,v17.16b,v18.16b,v19.16b}, [x7], x1        b.le            0f        \type           v4, v0,  v1,  v2,  v3        b               64b1280:        add             x7,  x0,  #64128:        \type           v5,  v0,  v1,  v2,  v3        \type           v6,  v0,  v1,  v2,  v3        \type           v7,  v0,  v1,  v2,  v3        \type           v16, v0,  v1,  v2,  v3        \type           v17, v0,  v1,  v2,  v3        st1             {v4.16b,v5.16b,v6.16b,v7.16b}, [x0], x1        \type           v18, v0,  v1,  v2,  v3        \type           v19, v0,  v1,  v2,  v3        subs            w5,  w5,  #1        st1             {v16.16b,v17.16b,v18.16b,v19.16b}, [x7], x1        b.le            0f        \type           v4, v0,  v1,  v2,  v3        b               128b0:        retL(\type\()_tbl):        .hword L(\type\()_tbl) - 1280b        .hword L(\type\()_tbl) -  640b        .hword L(\type\()_tbl) -  320b        .hword L(\type\()_tbl) -   16b        .hword L(\type\()_tbl) -   80b        .hword L(\type\()_tbl) -   40bendfunc.endm bidir_fn avgbidir_fn w_avgbidir_fn mask  .macro w_mask_fn typefunction w_mask_\type\()_8bpc_neon, export=1        clz             w8,  w4        adr             x9,  L(w_mask_\type\()_tbl)        sub             w8,  w8,  #24        ldrh            w8,  [x9,  x8,  lsl #1]        sub             x9,  x9,  w8,  uxtw        mov             w10, #6903        dup             v0.8h,   w10.if \type == 444        movi            v1.16b,  #64.elseif \type == 422        dup             v2.8b,   w7        movi            v3.8b,   #129        sub             v3.8b,   v3.8b,   v2.8b.elseif \type == 420        dup             v2.8h,   w7        movi            v3.8h,   #1, lsl #8        sub             v3.8h,   v3.8h,   v2.8h.endif        add             x12,  x0,  x1        lsl             x1,   x1,  #1        br              x94:        ld1             {v4.8h,   v5.8h},   [x2],  #32  // tmp1 (four rows at once)        ld1             {v6.8h,   v7.8h},   [x3],  #32  // tmp2 (four rows at once)        subs            w5,  w5,  #4        sub             v16.8h,  v6.8h,   v4.8h        sub             v17.8h,  v7.8h,   v5.8h        sabd            v18.8h,  v4.8h,   v6.8h        sabd            v19.8h,  v5.8h,   v7.8h        uqsub           v18.8h,  v0.8h,   v18.8h        uqsub           v19.8h,  v0.8h,   v19.8h        ushr            v18.8h,  v18.8h,  #8        ushr            v19.8h,  v19.8h,  #8        shl             v20.8h,  v18.8h,  #9        shl             v21.8h,  v19.8h,  #9        sqdmulh         v20.8h,  v20.8h,  v16.8h        sqdmulh         v21.8h,  v21.8h,  v17.8h        add             v20.8h,  v20.8h,  v4.8h        add             v21.8h,  v21.8h,  v5.8h        sqrshrun        v22.8b,  v20.8h,  #4        sqrshrun        v23.8b,  v21.8h,  #4.if \type == 444        xtn             v18.8b,   v18.8h        xtn2            v18.16b,  v19.8h        sub             v18.16b,  v1.16b,  v18.16b        st1             {v18.16b}, [x6],  #16.elseif \type == 422        addp            v18.8h,   v18.8h,  v19.8h        xtn             v18.8b,   v18.8h        uhsub           v18.8b,   v3.8b,   v18.8b        st1             {v18.8b},  [x6],  #8.elseif \type == 420        trn1            v24.2d,   v18.2d,  v19.2d        trn2            v25.2d,   v18.2d,  v19.2d        add             v24.8h,   v24.8h,  v25.8h        addp            v18.8h,   v24.8h,  v24.8h        sub             v18.4h,   v3.4h,   v18.4h        rshrn           v18.8b,   v18.8h,  #2        st1             {v18.s}[0],  [x6],  #4.endif        st1             {v22.s}[0],  [x0],  x1        st1             {v22.s}[1],  [x12], x1        st1             {v23.s}[0],  [x0],  x1        st1             {v23.s}[1],  [x12], x1        b.gt            4b        ret8:        ld1             {v4.8h,   v5.8h},   [x2],  #32        ld1             {v6.8h,   v7.8h},   [x3],  #32        subs            w5,  w5,  #2        sub             v16.8h,  v6.8h,   v4.8h        sub             v17.8h,  v7.8h,   v5.8h        sabd            v18.8h,  v4.8h,   v6.8h        sabd            v19.8h,  v5.8h,   v7.8h        uqsub           v18.8h,  v0.8h,   v18.8h        uqsub           v19.8h,  v0.8h,   v19.8h        ushr            v18.8h,  v18.8h,  #8        ushr            v19.8h,  v19.8h,  #8        shl             v20.8h,  v18.8h,  #9        shl             v21.8h,  v19.8h,  #9        sqdmulh         v20.8h,  v20.8h,  v16.8h        sqdmulh         v21.8h,  v21.8h,  v17.8h        add             v20.8h,  v20.8h,  v4.8h        add             v21.8h,  v21.8h,  v5.8h        sqrshrun        v22.8b,  v20.8h,  #4        sqrshrun        v23.8b,  v21.8h,  #4.if \type == 444        xtn             v18.8b,  v18.8h        xtn2            v18.16b, v19.8h        sub             v18.16b, v1.16b,  v18.16b        st1             {v18.16b}, [x6],  #16.elseif \type == 422        addp            v18.8h,  v18.8h,  v19.8h        xtn             v18.8b,  v18.8h        uhsub           v18.8b,  v3.8b,   v18.8b        st1             {v18.8b},  [x6],  #8.elseif \type == 420        add             v18.8h,  v18.8h,  v19.8h        addp            v18.8h,  v18.8h,  v18.8h        sub             v18.4h,  v3.4h,   v18.4h        rshrn           v18.8b,  v18.8h,  #2        st1             {v18.s}[0],  [x6],  #4.endif        st1             {v22.8b},  [x0],  x1        st1             {v23.8b},  [x12], x1        b.gt            8b        ret1280:640:320:160:        mov             w11, w4        sub             x1,  x1,  w4,  uxtw.if \type == 444        add             x10, x6,  w4,  uxtw.elseif \type == 422        add             x10, x6,  x11, lsr #1.endif        add             x9,  x3,  w4,  uxtw #1        add             x7,  x2,  w4,  uxtw #1161:        mov             w8,  w416:        ld1             {v4.8h,   v5.8h},   [x2],  #32        ld1             {v6.8h,   v7.8h},   [x3],  #32        ld1             {v16.8h,  v17.8h},  [x7],  #32        ld1             {v18.8h,  v19.8h},  [x9],  #32        subs            w8,  w8,  #16        sub             v6.8h,   v6.8h,   v4.8h        sub             v7.8h,   v7.8h,   v5.8h        sub             v18.8h,  v18.8h,  v16.8h        sub             v19.8h,  v19.8h,  v17.8h        abs             v20.8h,  v6.8h        abs             v21.8h,  v7.8h        abs             v22.8h,  v18.8h        abs             v23.8h,  v19.8h        uqsub           v20.8h,  v0.8h,   v20.8h        uqsub           v21.8h,  v0.8h,   v21.8h        uqsub           v22.8h,  v0.8h,   v22.8h        uqsub           v23.8h,  v0.8h,   v23.8h        ushr            v20.8h,  v20.8h,  #8        ushr            v21.8h,  v21.8h,  #8        ushr            v22.8h,  v22.8h,  #8        ushr            v23.8h,  v23.8h,  #8        shl             v24.8h,  v20.8h,  #9        shl             v25.8h,  v21.8h,  #9        shl             v26.8h,  v22.8h,  #9        shl             v27.8h,  v23.8h,  #9        sqdmulh         v24.8h,  v24.8h,  v6.8h        sqdmulh         v25.8h,  v25.8h,  v7.8h        sqdmulh         v26.8h,  v26.8h,  v18.8h        sqdmulh         v27.8h,  v27.8h,  v19.8h        add             v24.8h,  v24.8h,  v4.8h        add             v25.8h,  v25.8h,  v5.8h        add             v26.8h,  v26.8h,  v16.8h        add             v27.8h,  v27.8h,  v17.8h        sqrshrun        v24.8b,  v24.8h,  #4        sqrshrun        v25.8b,  v25.8h,  #4        sqrshrun        v26.8b,  v26.8h,  #4        sqrshrun        v27.8b,  v27.8h,  #4.if \type == 444        xtn             v20.8b,  v20.8h        xtn2            v20.16b, v21.8h        xtn             v21.8b,  v22.8h        xtn2            v21.16b, v23.8h        sub             v20.16b, v1.16b,  v20.16b        sub             v21.16b, v1.16b,  v21.16b        st1             {v20.16b}, [x6],  #16        st1             {v21.16b}, [x10], #16.elseif \type == 422        addp            v20.8h,  v20.8h,  v21.8h        addp            v21.8h,  v22.8h,  v23.8h        xtn             v20.8b,  v20.8h        xtn             v21.8b,  v21.8h        uhsub           v20.8b,  v3.8b,   v20.8b        uhsub           v21.8b,  v3.8b,   v21.8b        st1             {v20.8b},  [x6],  #8        st1             {v21.8b},  [x10], #8.elseif \type == 420        add             v20.8h,  v20.8h,  v22.8h        add             v21.8h,  v21.8h,  v23.8h        addp            v20.8h,  v20.8h,  v21.8h        sub             v20.8h,  v3.8h,   v20.8h        rshrn           v20.8b,  v20.8h,  #2        st1             {v20.8b},  [x6],  #8.endif        st1             {v24.8b,  v25.8b},  [x0],  #16        st1             {v26.8b,  v27.8b},  [x12], #16        b.gt            16b        subs            w5,  w5,  #2        add             x2,  x2,  w4,  uxtw #1        add             x3,  x3,  w4,  uxtw #1        add             x7,  x7,  w4,  uxtw #1        add             x9,  x9,  w4,  uxtw #1.if \type == 444        add             x6,  x6,  w4,  uxtw        add             x10, x10, w4,  uxtw.elseif \type == 422        add             x6,  x6,  x11, lsr #1        add             x10, x10, x11, lsr #1.endif        add             x0,  x0,  x1        add             x12, x12, x1        b.gt            161b        retL(w_mask_\type\()_tbl):        .hword L(w_mask_\type\()_tbl) - 1280b        .hword L(w_mask_\type\()_tbl) -  640b        .hword L(w_mask_\type\()_tbl) -  320b        .hword L(w_mask_\type\()_tbl) -  160b        .hword L(w_mask_\type\()_tbl) -    8b        .hword L(w_mask_\type\()_tbl) -    4bendfunc.endm w_mask_fn 444w_mask_fn 422w_mask_fn 420  function blend_8bpc_neon, export=1        adr             x6,  L(blend_tbl)        clz             w3,  w3        sub             w3,  w3,  #26        ldrh            w3,  [x6,  x3,  lsl #1]        sub             x6,  x6,  w3,  uxtw        movi            v4.16b,  #64        add             x8,  x0,  x1        lsl             x1,  x1,  #1        br              x64:        ld1             {v2.8b},     [x5],  #8        ld1             {v1.d}[0],   [x2],  #8        ld1             {v0.s}[0],   [x0]        subs            w4,  w4,  #2        ld1             {v0.s}[1],   [x8]        sub             v3.8b,   v4.8b,   v2.8b        umull           v5.8h,   v1.8b,   v2.8b        umlal           v5.8h,   v0.8b,   v3.8b        rshrn           v6.8b,   v5.8h,   #6        st1             {v6.s}[0],   [x0],  x1        st1             {v6.s}[1],   [x8],  x1        b.gt            4b        ret8:        ld1             {v2.16b},  [x5],  #16        ld1             {v1.16b},  [x2],  #16        ld1             {v0.d}[0],   [x0]        ld1             {v0.d}[1],   [x8]        sub             v3.16b,  v4.16b,  v2.16b        subs            w4,  w4,  #2        umull           v5.8h,   v1.8b,   v2.8b        umlal           v5.8h,   v0.8b,   v3.8b        umull2          v6.8h,   v1.16b,  v2.16b        umlal2          v6.8h,   v0.16b,  v3.16b        rshrn           v7.8b,   v5.8h,   #6        rshrn2          v7.16b,  v6.8h,   #6        st1             {v7.d}[0],   [x0],  x1        st1             {v7.d}[1],   [x8],  x1        b.gt            8b        ret16:        ld1             {v1.16b,  v2.16b},  [x5],  #32        ld1             {v5.16b,  v6.16b},  [x2],  #32        ld1             {v0.16b},  [x0]        subs            w4,  w4,  #2        sub             v7.16b,  v4.16b,  v1.16b        sub             v20.16b, v4.16b,  v2.16b        ld1             {v3.16b},  [x8]        umull           v16.8h,  v5.8b,   v1.8b        umlal           v16.8h,  v0.8b,   v7.8b        umull2          v17.8h,  v5.16b,  v1.16b        umlal2          v17.8h,  v0.16b,  v7.16b        umull           v21.8h,  v6.8b,   v2.8b        umlal           v21.8h,  v3.8b,   v20.8b        umull2          v22.8h,  v6.16b,  v2.16b        umlal2          v22.8h,  v3.16b,  v20.16b        rshrn           v18.8b,  v16.8h,  #6        rshrn2          v18.16b, v17.8h,  #6        rshrn           v19.8b,  v21.8h,  #6        rshrn2          v19.16b, v22.8h,  #6        st1             {v18.16b}, [x0],  x1        st1             {v19.16b}, [x8],  x1        b.gt            16b        ret32:        ld1             {v0.16b,  v1.16b,  v2.16b,  v3.16b},  [x5],  #64        ld1             {v16.16b, v17.16b, v18.16b, v19.16b}, [x2],  #64        ld1             {v20.16b, v21.16b}, [x0]        subs            w4,  w4,  #2        ld1             {v22.16b, v23.16b}, [x8]        sub             v5.16b,  v4.16b,  v0.16b        sub             v6.16b,  v4.16b,  v1.16b        sub             v30.16b, v4.16b,  v2.16b        sub             v31.16b, v4.16b,  v3.16b

Lines 1–500 of 3247. Load more