;*****************************************************************************;* x86-optimized functions for convolution filter;*;* This file is part of FFmpeg.;*;* FFmpeg is free software; you can redistribute it and/or;* modify it under the terms of the GNU Lesser General Public;* License as published by the Free Software Foundation; either;* version 2.1 of the License, or (at your option) any later version.;*;* FFmpeg is distributed in the hope that it will be useful,;* but WITHOUT ANY WARRANTY; without even the implied warranty of;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU;* Lesser General Public License for more details.;*;* You should have received a copy of the GNU Lesser General Public;* License along with FFmpeg; if not, write to the Free Software;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA;****************************************************************************** %include "libavutil/x86/x86util.asm" SECTION_RODATAhalf:   dd 0.5data_p1: dd  1data_n1: dd -1data_p2: dd  2data_n2: dd -2 ALIGN 64sobel_perm: db  0, 16, 32, 48,  1, 17, 33, 49,  2, 18, 34, 50,  3, 19, 35, 51            db  4, 20, 36, 52,  5, 21, 37, 53,  6, 22, 38, 54,  7, 23, 39, 55            db  8, 24, 40, 56,  9, 25, 41, 57, 10, 26, 42, 58, 11, 27, 43, 59            db 12, 28, 44, 60, 13, 29, 45, 61, 14, 30, 46, 62, 15, 31, 47, 63sobel_mulA: db -1,  1, -2,  2sobel_mulB: db  1, -1,  2, -2 SECTION .text ; void filter_3x3_sse4(uint8_t *dst, int width,;                      float rdiv, float bias, const int *const matrix,;                      const uint8_t *c[], int peak, int radius,;                      int dstride, int stride)  %macro PROCESS_V 1    movss m2, [matrixq + 4 * %1]    VBROADCASTSS m2, m2    movss m3, [c%1q + xq]    punpcklbw m3, m6    punpcklwd m3, m6    pmulld m2, m3    paddd m4, m2%endmacro %macro PROCESS_S 1    movzx ptrd, byte [c%1q + xq]    imul  ptrd, [matrixq + 4 * %1]    add   rd, ptrd%endmacro %macro FILTER_3X3 0%if UNIX64cglobal filter_3x3, 4, 15, 7, dst, width, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x%elsecglobal filter_3x3, 4, 15, 7, dst, width, rdiv, bias, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x%endif %if WIN64    SWAP m0, m2    SWAP m1, m3    mov  r2q, matrixmp    mov  r3q, ptrmp    DEFINE_ARGS dst, width, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x%endif    movsxdifnidn widthq, widthd    VBROADCASTSS m0, m0    VBROADCASTSS m1, m1    pxor  m6, m6    movss m5, [half]    VBROADCASTSS m5, m5    mov   c0q, [ptrq + 0*gprsize]    mov   c1q, [ptrq + 1*gprsize]    mov   c2q, [ptrq + 2*gprsize]    mov   c3q, [ptrq + 3*gprsize]    mov   c4q, [ptrq + 4*gprsize]    mov   c5q, [ptrq + 5*gprsize]    mov   c6q, [ptrq + 6*gprsize]    mov   c7q, [ptrq + 7*gprsize]    mov   c8q, [ptrq + 8*gprsize]     xor   xq, xq    cmp   widthq, mmsize/4    jl .loop2     mov   rq, widthq    and   rq, mmsize/4-1    sub   widthq, rq .loop1:    pxor m4, m4         ; sum = 0;     PROCESS_V 0    PROCESS_V 1    PROCESS_V 2    PROCESS_V 3    PROCESS_V 4    PROCESS_V 5    PROCESS_V 6    PROCESS_V 7    PROCESS_V 8     cvtdq2ps  m4, m4    mulps     m4, m0     ; sum *= rdiv    addps     m4, m1     ; sum += bias    addps     m4, m5     ; sum += 0.5    cvttps2dq m4, m4    packssdw  m4, m4    packuswb  m4, m4    movss     [dstq + xq], m4     add xq, mmsize/4    cmp xq, widthq    jl .loop1     add widthq, rq    cmp xq, widthq    jge .end .loop2:    ; reuse r to hold sum, init with zero    xor rd, rd     PROCESS_S 0    PROCESS_S 1    PROCESS_S 2    PROCESS_S 3    PROCESS_S 4    PROCESS_S 5    PROCESS_S 6    PROCESS_S 7    PROCESS_S 8     pxor      m4, m4    cvtsi2ss  m4, rd    mulss     m4, m0     ; sum *= rdiv    addss     m4, m1     ; sum += bias    addss     m4, m5     ; sum += 0.5    ; we don't have simple scalar instructions to convert    ; from 32bit to 8bit with saturation, so here    ; just use packed version SSE instructions for simplicity.    cvttps2dq m4, m4     ; trunc to integer    packssdw  m4, m4    packuswb  m4, m4    movd      rd, m4    mov       [dstq + xq], rb     add xq, 1    cmp xq, widthq    jl .loop2.end:    RET%endmacro %if ARCH_X86_64INIT_XMM sse4FILTER_3X3%endif %macro SOBEL_MUL 2    movzx ptrd, byte [c%1q + xq]    imul  ptrd, [%2]    add   rd, ptrd%endmacro %macro SOBEL_ADD 1    movzx ptrd, byte [c%1q + xq]    add   rd, ptrd%endmacro ; void filter_sobel_avx512(uint8_t *dst, int width,;                      float scale, float delta, const int *const matrix,;                      const uint8_t *c[], int peak, int radius,;                      int dstride, int stride)%macro FILTER_SOBEL 0%if UNIX64cglobal filter_sobel, 4, 15, 7, dst, width, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x%elsecglobal filter_sobel, 4, 15, 7, dst, width, rdiv, bias, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x%endif%if WIN64    VBROADCASTSS m0, xmm2    VBROADCASTSS m1, xmm3    mov  r2q, matrixmp    mov  r3q, ptrmp    DEFINE_ARGS dst, width, matrix, ptr, c0, c1, c2, c3, c4, c5, c6, c7, c8, r, x%else    VBROADCASTSS m0, xmm0    VBROADCASTSS m1, xmm1%endif    movsxdifnidn widthq, widthd    pxor  m6, m6    mov   c0q, [ptrq + 0*gprsize]    mov   c1q, [ptrq + 1*gprsize]    mov   c2q, [ptrq + 2*gprsize]    mov   c3q, [ptrq + 3*gprsize]    mov   c4q, [ptrq + 4*gprsize]    mov   c5q, [ptrq + 5*gprsize]    mov   c6q, [ptrq + 6*gprsize]    mov   c7q, [ptrq + 7*gprsize]    mov   c8q, [ptrq + 8*gprsize]     xor   xq, xq    cmp   widthq, mmsize/4    jl .loop2     mov   rq, widthq    and   rq, mmsize/4-1    sub   widthq, rq     mova  m6, [sobel_perm].loop1:    movu          xm3, [c2q + xq]    pmovzxbd      m5, [c0q + xq]    vinserti32x4  ym3, [c6q + xq], 1    pmovzxbd      m4, [c8q + xq]    vinserti32x4  m2, m3, [c1q + xq], 2    vinserti32x4  m3, [c5q + xq], 2    vinserti32x4  m2, [c7q + xq], 3    vinserti32x4  m3, [c3q + xq], 3    vpermb        m2, m6, m2    psubd         m4, m5    vpermb        m3, m6, m3    mova          m5, m4    vpdpbusd      m4, m2, [sobel_mulA] {1to16}    vpdpbusd      m5, m3, [sobel_mulB] {1to16}     cvtdq2ps  m4, m4    mulps     m4, m4     cvtdq2ps    m5, m5    VFMADD231PS m4, m5, m5     sqrtps    m4, m4    fmaddps m4, m4, m0, m1    cvttps2dq m4, m4    vpmovusdb [dstq + xq], m4     add xq, mmsize/4    cmp xq, widthq    jl .loop1     add widthq, rq    cmp xq, widthq    jge .end .loop2:    xor  rd, rd    pxor m4, m4     ;Gx    SOBEL_MUL 0, data_n1    SOBEL_MUL 1, data_n2    SOBEL_MUL 2, data_n1    SOBEL_ADD 6    SOBEL_MUL 7, data_p2    SOBEL_ADD 8     cvtsi2ss xmm4, rd    mulss    xmm4, xmm4     xor rd, rd    ;Gy    SOBEL_MUL 0, data_n1    SOBEL_ADD 2    SOBEL_MUL 3, data_n2    SOBEL_MUL 5, data_p2    SOBEL_MUL 6, data_n1    SOBEL_ADD 8     cvtsi2ss  xmm5, rd    fmaddss xmm4, xmm5, xmm5, xmm4     sqrtps    xmm4, xmm4    fmaddss   xmm4, xmm4, xm0, xm1     ;sum = sum * rdiv + bias    cvttps2dq xmm4, xmm4     ; trunc to integer    packssdw  xmm4, xmm4    packuswb  xmm4, xmm4    movd      rd, xmm4    mov       [dstq + xq], rb     add xq, 1    cmp xq, widthq    jl .loop2.end:    RET%endmacro %if ARCH_X86_64%if HAVE_AVX512ICL_EXTERNALINIT_ZMM avx512iclFILTER_SOBEL%endif%endif