/* * Copyright © 2018, VideoLAN and dav1d authors * Copyright © 2018, Janne Grunau * Copyright © 2018, Martin Storsjo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this *    list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, *    this list of conditions and the following disclaimer in the documentation *    and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "src/arm/asm.S"#include "util.S" .macro avg dst0, dst1, t0, t1, t2, t3        vld1.16         {\t0,\t1},   [r2, :128]!        vld1.16         {\t2,\t3},   [r3, :128]!        vadd.i16        \t0,   \t0,  \t2        vadd.i16        \t1,   \t1,  \t3        vqrshrun.s16    \dst0, \t0,  #5        vqrshrun.s16    \dst1, \t1,  #5.endm .macro w_avg dst0, dst1, t0, t1, t2, t3        vld1.16         {\t0,\t1},   [r2, :128]!        vld1.16         {\t2,\t3},   [r3, :128]!        vsub.i16        \t0,   \t2,  \t0        vsub.i16        \t1,   \t3,  \t1        vqdmulh.s16     \t0,   \t0,  q15        vqdmulh.s16     \t1,   \t1,  q15        vadd.i16        \t0,   \t2,  \t0        vadd.i16        \t1,   \t3,  \t1        vqrshrun.s16    \dst0, \t0,  #4        vqrshrun.s16    \dst1, \t1,  #4.endm .macro mask dst0, dst1, t0, t1, t2, t3        vld1.8          {q14}, [lr, :128]!        vld1.16         {\t0,\t1},   [r2, :128]!        vmul.i8         q14,   q14,  q15        vld1.16         {\t2,\t3},   [r3, :128]!        vshll.i8        q13,   d28,  #8        vshll.i8        q14,   d29,  #8        vsub.i16        \t0,   \t2,  \t0        vsub.i16        \t1,   \t3,  \t1        vqdmulh.s16     \t0,   \t0,  q13        vqdmulh.s16     \t1,   \t1,  q14        vadd.i16        \t0,   \t2,  \t0        vadd.i16        \t1,   \t3,  \t1        vqrshrun.s16    \dst0, \t0,  #4        vqrshrun.s16    \dst1, \t1,  #4.endm .macro bidir_fn typefunction \type\()_8bpc_neon, export=1        push            {r4-r6,lr}        ldr             r4, [sp, #16]        ldr             r5, [sp, #20]        clz             r4,  r4.ifnc \type, avg        ldr             lr, [sp, #24].endif.ifc \type, w_avg        vdup.s16        q15, lr        vneg.s16        q15, q15        vshl.i16        q15, q15, #11.endif.ifc \type, mask        vmov.i8         q15, #256-2.endif        adr             r12, L(\type\()_tbl)        sub             r4,  r4,  #24        ldr             r4,  [r12, r4, lsl #2]        \type           d16, d17, q0,  q1,  q2,  q3        add             r12, r12, r4        bx              r12         .align 2L(\type\()_tbl):        .word 1280f - L(\type\()_tbl) + CONFIG_THUMB        .word 640f  - L(\type\()_tbl) + CONFIG_THUMB        .word 320f  - L(\type\()_tbl) + CONFIG_THUMB        .word 160f  - L(\type\()_tbl) + CONFIG_THUMB        .word 80f   - L(\type\()_tbl) + CONFIG_THUMB        .word 4f    - L(\type\()_tbl) + CONFIG_THUMB 4:        add             r6,  r0,  r1        lsl             r1,  r1,  #1        cmp             r5,  #4        vst1.32         {d16[0]},  [r0, :32], r1        vst1.32         {d16[1]},  [r6, :32], r1        vst1.32         {d17[0]},  [r0, :32], r1        vst1.32         {d17[1]},  [r6, :32], r1        beq             0f        \type           d18, d19,  q0,  q1,  q2,  q3        cmp             r5,  #8        vst1.32         {d18[0]},  [r0, :32], r1        vst1.32         {d18[1]},  [r6, :32], r1        vst1.32         {d19[0]},  [r0, :32], r1        vst1.32         {d19[1]},  [r6, :32], r1        beq             0f        \type           d16, d17, q0,  q1,  q2,  q3        vst1.32         {d16[0]},  [r0, :32], r1        vst1.32         {d16[1]},  [r6, :32], r1        \type           d18, d19,  q0,  q1,  q2,  q3        vst1.32         {d17[0]},  [r0, :32], r1        vst1.32         {d17[1]},  [r6, :32], r1        vst1.32         {d18[0]},  [r0, :32], r1        vst1.32         {d18[1]},  [r6, :32], r1        vst1.32         {d19[0]},  [r0, :32], r1        vst1.32         {d19[1]},  [r6, :32], r1        pop             {r4-r6,pc}80:        add             r6,  r0,  r1        lsl             r1,  r1,  #18:        vst1.8          {d16},  [r0, :64], r1        \type           d18, d19, q0,  q1,  q2,  q3        vst1.8          {d17},  [r6, :64], r1        vst1.8          {d18},  [r0, :64], r1        subs            r5,  r5,  #4        vst1.8          {d19},  [r6, :64], r1        ble             0f        \type           d16, d17, q0,  q1,  q2,  q3        b               8b160:        add             r6,  r0,  r1        lsl             r1,  r1,  #116:        \type           d18, d19, q0, q1, q2, q3        vst1.8          {q8},  [r0, :128], r1        \type           d20, d21, q0, q1, q2, q3        vst1.8          {q9},  [r6, :128], r1        \type           d22, d23, q0, q1, q2, q3        vst1.8          {q10}, [r0, :128], r1        subs            r5,  r5,  #4        vst1.8          {q11}, [r6, :128], r1        ble             0f        \type           d16, d17, q0, q1, q2, q3        b               16b320:        add             r6,  r0,  r1        lsl             r1,  r1,  #132:        \type           d18, d19, q0, q1, q2, q3        \type           d20, d21, q0, q1, q2, q3        vst1.8          {q8,  q9},  [r0, :128], r1        \type           d22, d23, q0, q1, q2, q3        subs            r5,  r5,  #2        vst1.8          {q10, q11}, [r6, :128], r1        ble             0f        \type           d16, d17, q0, q1, q2, q3        b               32b640:        add             r6,  r0,  #3264:        \type           d18, d19, q0, q1, q2, q3        \type           d20, d21, q0, q1, q2, q3        \type           d22, d23, q0, q1, q2, q3        vst1.8          {q8,  q9},  [r0, :128], r1        \type           d16, d17, q0, q1, q2, q3        vst1.8          {q10, q11}, [r6, :128], r1        \type           d18, d19, q0, q1, q2, q3        \type           d20, d21, q0, q1, q2, q3        vst1.8          {q8,  q9},  [r0, :128], r1        \type           d22, d23, q0, q1, q2, q3        subs            r5,  r5,  #2        vst1.8          {q10, q11}, [r6, :128], r1        ble             0f        \type           d16, d17, q0, q1, q2, q3        b               64b1280:        sub             r1,  r1,  #32        add             r6,  r0,  #64128:        \type           d18, d19, q0, q1, q2, q3        \type           d20, d21, q0, q1, q2, q3        \type           d22, d23, q0, q1, q2, q3        vst1.8          {q8,  q9},  [r0, :128]!        \type           d16, d17, q0, q1, q2, q3        vst1.8          {q10, q11}, [r0, :128], r1        \type           d18, d19, q0, q1, q2, q3        \type           d20, d21, q0, q1, q2, q3        vst1.8          {q8,  q9},  [r6, :128]!        \type           d22, d23, q0, q1, q2, q3        subs            r5,  r5,  #1        vst1.8          {q10, q11}, [r6, :128], r1        ble             0f        \type           d16, d17, q0, q1, q2, q3        b               128b 0:        pop             {r4-r6,pc}endfunc.endm bidir_fn avgbidir_fn w_avgbidir_fn mask  .macro w_mask_fn typefunction w_mask_\type\()_8bpc_neon, export=1        push            {r4-r9,lr}        ldr             r4,  [sp, #28]        ldr             r5,  [sp, #32]        ldr             r6,  [sp, #36]        ldr             r7,  [sp, #40]        clz             r8,  r4        adr             r9,  L(w_mask_\type\()_tbl)        sub             r8,  r8,  #24        ldr             r8,  [r9,  r8,  lsl #2]        add             r9,  r9,  r8        movw            r12, #6903        vdup.16         q14, r12.if \type == 444        vmov.i8         q15, #64.elseif \type == 422        vdup.8          d0,  r7         // d0[] <- sign        vmov.i8         d30, #129        vsub.i8         d30, d30, d0    // 129 - sign.elseif \type == 420        vdup.16         q0,  r7         // d0[] <- sign        vmov.i16        q15, #256        vsub.i16        q15, q15, q0    // 256 - sign.endif        add             r12, r0,  r1        lsl             r1,  r1,  #1        bx              r9         .align 2L(w_mask_\type\()_tbl):        .word 1280f  - L(w_mask_\type\()_tbl) + CONFIG_THUMB        .word 640f   - L(w_mask_\type\()_tbl) + CONFIG_THUMB        .word 320f   - L(w_mask_\type\()_tbl) + CONFIG_THUMB        .word 160f   - L(w_mask_\type\()_tbl) + CONFIG_THUMB        .word 8f     - L(w_mask_\type\()_tbl) + CONFIG_THUMB        .word 4f     - L(w_mask_\type\()_tbl) + CONFIG_THUMB 4:        vld1.16         {d0,  d1,  d2,  d3},  [r2,  :128]! // tmp1 (four rows at once)        vld1.16         {d4,  d5,  d6,  d7},  [r3,  :128]! // tmp2 (four rows at once)        subs            r5,  r5,  #4        vsub.i16        q8,  q2,  q0    // tmp2-tmp1        vsub.i16        q9,  q3,  q1        vabd.s16        q10, q0,  q2    // (abs(tmp1[x] - tmp2[x]))        vabd.s16        q11, q1,  q3        vqsub.u16       q10, q14, q10   // 6903 - abs ()        vqsub.u16       q11, q14, q11        vshr.s16        q10, q10, #8    // 64-m = (6903 - abs()) >> 8        vshr.s16        q11, q11, #8        vshl.s16        q12, q10, #9    // (64-m)<<9        vshl.s16        q13, q11, #9        vqdmulh.s16     q12, q12, q8    // ((tmp2-tmp1)*(64-m)<<9)>>15        vqdmulh.s16     q13, q13, q9        vadd.i16        q12, q12, q0    // (((tmp2-tmp1)*(64-m)<<9)>>15) + tmp1        vadd.i16        q13, q13, q1        vqrshrun.s16    d24, q12, #4    // (((((tmp2-tmp1)*(64-m)<<9)>>15) + tmp1) + 8) >> 4        vqrshrun.s16    d25, q13, #4.if \type == 444        vmovn.u16       d20, q10        // 64 - m        vmovn.u16       d21, q11        vsub.i8         q10, q15, q10   // m        vst1.8          {d20, d21}, [r6,  :128]!.elseif \type == 422        vpadd.s16       d20, d20, d21   // (64 - m) + (64 - n) (column wise addition)        vpadd.s16       d21, d22, d23        vmovn.s16       d6,  q10        vhsub.u8        d6,  d30, d6    // ((129 - sign) - ((64 - m) + (64 - n))) >> 1        vst1.8          {d6},  [r6,  :64]!.elseif \type == 420        vadd.s16        d20, d20, d21   // (64 - my1) + (64 - my2) (row wise addition)        vadd.s16        d21, d22, d23        vpadd.s16       d20, d20, d21   // (128 - m) + (128 - n) (column wise addition)        vsub.s16        d20, d30, d20   // (256 - sign) - ((128 - m) + (128 - n))        vrshrn.u16      d20, q10,  #2   // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2        vst1.32         {d20[0]}, [r6,  :32]!.endif        vst1.32         {d24[0]}, [r0,  :32], r1        vst1.32         {d24[1]}, [r12, :32], r1        vst1.32         {d25[0]}, [r0,  :32], r1        vst1.32         {d25[1]}, [r12, :32], r1        bgt             4b        pop             {r4-r9,pc}8:        vld1.16         {d0,  d1,  d2,  d3},  [r2,  :128]! // tmp1y1, tmp1y2        vld1.16         {d4,  d5,  d6,  d7},  [r3,  :128]! // tmp2y1, tmp2y2        subs            r5,  r5,  #2        vsub.i16        q8,  q2,  q0    // tmp2y1 - tmp1y1        vsub.i16        q9,  q3,  q1    // tmp2y2 - tmp1y2        vabd.s16        q10, q0,  q2    // abs(tmp1y1 - tmp2y1)        vabd.s16        q11, q1,  q3    // abs(tmp1y2 - tmp2y2)        vqsub.u16       q10, q14, q10   // 6903 - abs(tmp1y1 - tmp2y1)        vqsub.u16       q11, q14, q11   // 6903 - abs(tmp1y2 - tmp2y2)        vshr.s16        q10, q10, #8    // 64 - my1 = 6903 - abs(tmp1y1 - tmp2y1) >> 8        vshr.s16        q11, q11, #8    // 64 - my2 = 6903 - abs(tmp1y2 - tmp2y2) >> 8        vshl.s16        q12, q10, #9    // (64 - my1) << 9        vshl.s16        q13, q11, #9    // (64 - my2) << 9        vqdmulh.s16     q12, q12, q8    // ((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15        vqdmulh.s16     q13, q13, q9    // ((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15        vadd.s16        q12, q12, q0    // (((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1        vadd.s16        q13, q13, q1    // (((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2        vqrshrun.s16    d24, q12, #4    // (((((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1) + 8) >> 4        vqrshrun.s16    d25, q13, #4    // (((((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2) + 8) >> 4.if \type == 444        vmovn.u16       d20, q10        // 64 - m        vmovn.u16       d21, q11        vsub.i8         q10, q15, q10   // m        vst1.8          {d20, d21}, [r6,  :128]!.elseif \type == 422        vpadd.s16       d20, d20, d21   // (64 - my1) + (64 - ny1) (column wise addition)        vpadd.s16       d21, d22, d23   // (64 - my2) + (64 - ny2)        vmovn.s16       d20, q10        vhsub.u8        d20, d30, d20   // ((129 - sign) - ((64 - my1/y2) + (64 - ny1/y2))) >> 1        vst1.8          {d20}, [r6,  :64]!.elseif \type == 420        vadd.s16        q10, q10, q11   // (64 - my1) + (64 - my2) (row wise addition)        vpadd.s16       d20, d20, d21   // (128 - m) + (128 - n) (column wise addition)        vsub.s16        d20, d30, d20   // (256 - sign) - ((128 - m) + (128 - n))        vrshrn.u16      d20, q10, #2    // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2        vst1.32         {d20[0]}, [r6,  :32]!.endif        vst1.16         {d24}, [r0,  :64], r1        vst1.16         {d25}, [r12, :64], r1        bgt             8b        pop             {r4-r9,pc}1280:640:320:160:        sub             r1,  r1,  r4.if \type == 444        add             lr,  r6,  r4.elseif \type == 422        add             lr,  r6,  r4,  lsr #1.endif        add             r9,  r3,  r4,  lsl #1        add             r7,  r2,  r4,  lsl #1161:        mov             r8,  r416:        vld1.16         {d0,  d1,  d2,  d3},  [r2,  :128]! // tmp1y1        vld1.16         {d4,  d5,  d6,  d7},  [r3,  :128]! // tmp2y1        vld1.16         {d16, d17, d18, d19}, [r7,  :128]! // tmp1y2        subs            r8,  r8,  #16        vsub.i16        q2,  q2,  q0    // tmp2y1 - tmp1y1        vsub.i16        q3,  q3,  q1        vabs.s16        q10, q2         // abs(tm2y1 - tmp1y1)        vabs.s16        q11, q3        vqsub.u16       q10, q14, q10   // 6903 - abs(tmp1y1 - tmp2y1)        vqsub.u16       q11, q14, q11        vshr.s16        q10, q10, #8    // 64 - my1 = 6903 - abs(tmp1y1 - tmp2y1) >> 8        vshr.s16        q11, q11, #8        vshl.s16        q12, q10, #9    // (64 - my1) << 9        vshl.s16        q13, q11, #9        vqdmulh.s16     q12, q12, q2    // ((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15        vqdmulh.s16     q13, q13, q3        vadd.i16        q12, q12, q0    // (((tmp2y1 - tmp1y1) * (64 - my1) << 9) >> 15) + tmp1y1        vadd.i16        q13, q13, q1        vld1.16         {d0,  d1,  d2,  d3},  [r9,  :128]! // tmp2h2.if \type == 444        vmovn.u16       d20, q10        // 64 - my1        vmovn.u16       d21, q11        vsub.i8         q10, q15, q10   // my1        vst1.8          {d20, d21}, [r6,  :128]!.elseif \type == 422        vpadd.s16       d20, d20, d21   // (64 - my1) + (64 - ny1) (column wise addition)        vpadd.s16       d21, d22, d23        vmovn.s16       d20, q10        vhsub.u8        d20, d30, d20   // ((129 - sign) - ((64 - my1) + (64 - ny1))) >> 1        vst1.8          {d20}, [r6,  :64]!.endif        vqrshrun.s16    d24, q12, #4    // (((((tmp2y1 - tmp1y1)*(64 - my1) << 9) >> 15) + tmp1y1) + 8) >> 4        vqrshrun.s16    d25, q13, #4        vsub.i16        q0,  q0,  q8    // tmp2y2 - tmp1y2        vsub.i16        q1,  q1,  q9        vst1.16         {d24, d25}, [r0,  :128]!    // store dsty1        vabs.s16        q2,  q0         // abs(tmp2y2 - tmp1y2)        vabs.s16        q3,  q1        vqsub.u16       q2,  q14, q2    // 6903 - abs(tmp2y2 - tmp1y2)        vqsub.u16       q3,  q14, q3        vshr.s16        q2,  q2,  #8    // (6903 - abs(tmp2y2 - tmp1y2)) >> 8        vshr.s16        q3,  q3,  #8        vshl.s16        q12, q2,  #9    // (64 - my2) << 9        vshl.s16        q13, q3,  #9.if \type == 444        vmovn.u16       d4,  q2         // 64 - my2        vmovn.u16       d5,  q3        vsub.i8         q2,  q15, q2    // my2        vst1.8          {d4,  d5},  [lr,  :128]!.elseif \type == 422        vpadd.s16       d4,  d4,  d5    // (64 - my2) + (64 - ny2) (column wise addition)        vpadd.s16       d5,  d6,  d7        vmovn.s16       d4,  q2        vhsub.u8        d4,  d30, d4    // ((129 - sign) - ((64 - my2) + (64 - ny2))) >> 1        vst1.8          {d4},  [lr,  :64]!.elseif \type == 420        vadd.s16        q10, q10, q2    // (64 - my1) + (64 - my2) (row wise addition)        vadd.s16        q11, q11, q3        vpadd.s16       d20, d20, d21   // (128 - m) + (128 - n) (column wise addition)        vpadd.s16       d21, d22, d23        vsub.s16        q10, q15, q10   // (256 - sign) - ((128 - m) + (128 - n))        vrshrn.u16      d20, q10, #2    // ((256 - sign) - ((128 - m) + (128 - n)) + 2) >> 2        vst1.8          {d20}, [r6,  :64]!.endif        vqdmulh.s16     q12, q12, q0    // ((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15        vqdmulh.s16     q13, q13, q1        vadd.i16        q12, q12, q8    // (((tmp2y2 - tmp1y2) * (64 - my2) << 9) >> 15) + tmp1y2        vadd.i16        q13, q13, q9        vqrshrun.s16    d24, q12, #4    // (((((tmp2y2 - tmp1y2)*(64 - my2) << 9) >> 15) + tmp1y2) + 8) >> 4        vqrshrun.s16    d25, q13, #4        vst1.16         {d24, d25}, [r12, :128]!   // store dsty2        bgt             16b        subs            r5,  r5,  #2        add             r2,  r2,  r4,  lsl #1        add             r3,  r3,  r4,  lsl #1        add             r7,  r7,  r4,  lsl #1        add             r9,  r9,  r4,  lsl #1.if \type == 444        add             r6,  r6,  r4        add             lr,  lr,  r4.elseif \type == 422        add             r6,  r6,  r4,  lsr #1        add             lr,  lr,  r4,  lsr #1.endif        add             r0,  r0,  r1        add             r12, r12, r1        bgt             161b        pop             {r4-r9,pc}endfunc.endm w_mask_fn 444w_mask_fn 422w_mask_fn 420  function blend_8bpc_neon, export=1        push            {r4-r5,lr}        ldr             r4,  [sp, #12]        ldr             r5,  [sp, #16]        clz             lr,  r3        adr             r3,  L(blend_tbl)        sub             lr,  lr,  #26        ldr             lr,  [r3, lr, lsl #2]        add             r3,  r3,  lr        bx              r3         .align 2L(blend_tbl):        .word 320f  - L(blend_tbl) + CONFIG_THUMB        .word 160f  - L(blend_tbl) + CONFIG_THUMB        .word 80f   - L(blend_tbl) + CONFIG_THUMB        .word 40f   - L(blend_tbl) + CONFIG_THUMB 40:        vmov.i8         d22, #64        add             r12, r0,  r1        lsl             r1,  r1,  #14:        vld1.u8         {d2},     [r5,  :64]!        vld1.u8         {d1},     [r2,  :64]!        vld1.32         {d0[]},   [r0,  :32]        subs            r4,  r4,  #2        vld1.32         {d0[1]},  [r12, :32]        vsub.i8         d3,  d22, d2        vmull.u8        q8,  d1,  d2        vmlal.u8        q8,  d0,  d3        vrshrn.i16      d20, q8,  #6        vst1.32         {d20[0]}, [r0,  :32], r1        vst1.32         {d20[1]}, [r12, :32], r1        bgt             4b        pop             {r4-r5,pc}80:        vmov.i8         d16, #64        add             r12, r0,  r1        lsl             r1,  r1,  #18:        vld1.u8         {q1},  [r5,  :128]!        vld1.u8         {q2},  [r2,  :128]!        vld1.u8         {d0},  [r0,  :64]        vsub.i8         d17, d16, d2

Lines 1–500 of 3349. Load more