; Copyright © 2018, VideoLAN and dav1d authors; Copyright © 2018, Two Orioles, LLC; All rights reserved.;; Redistribution and use in source and binary forms, with or without; modification, are permitted provided that the following conditions are met:;; 1. Redistributions of source code must retain the above copyright notice, this;    list of conditions and the following disclaimer.;; 2. Redistributions in binary form must reproduce the above copyright notice,;    this list of conditions and the following disclaimer in the documentation;    and/or other materials provided with the distribution.;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND; ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR; ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES; (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND; ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. %include "config.asm"%include "ext/x86/x86inc.asm" %if ARCH_X86_64 SECTION_RODATA 64 %macro SMOOTH_WEIGHT_TABLE 1-*    %rep %0        db %1-128, 127-%1        %rotate 1    %endrep%endmacro ; sm_weights[], but modified to precalculate x and 256-x with offsets to; enable efficient use of pmaddubsw (which requires signed values)smooth_weights: SMOOTH_WEIGHT_TABLE         \      0,   0, 255, 128, 255, 149,  85,  64, \    255, 197, 146, 105,  73,  50,  37,  32, \    255, 225, 196, 170, 145, 123, 102,  84, \     68,  54,  43,  33,  26,  20,  17,  16, \    255, 240, 225, 210, 196, 182, 169, 157, \    145, 133, 122, 111, 101,  92,  83,  74, \     66,  59,  52,  45,  39,  34,  29,  25, \     21,  17,  14,  12,  10,   9,   8,   8, \    255, 248, 240, 233, 225, 218, 210, 203, \    196, 189, 182, 176, 169, 163, 156, 150, \    144, 138, 133, 127, 121, 116, 111, 106, \    101,  96,  91,  86,  82,  77,  73,  69, \     65,  61,  57,  54,  50,  47,  44,  41, \     38,  35,  32,  29,  27,  25,  22,  20, \     18,  16,  15,  13,  12,  10,   9,   8, \      7,   6,   6,   5,   5,   4,   4,   4 pb_1to32:     db  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16              db 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32pb_32to1:     db 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17pb_16to1:     db 16, 15, 14, 13, 12, 11, 10,  9,  8,  7,  6,  5,  4,  3,  2,  1z_filter_wh:  db  7,  7, 11, 11, 15, 15, 19, 19, 19, 23, 23, 23, 31, 31, 31, 39              db 39, 39, 47, 47, 47, 63, 63, 63, 79, 79, 79, -1z_filter_k:   db  0, 16,  0, 16,  0, 20,  0, 20,  8, 16,  8, 16              db 32, 16, 32, 16, 24, 20, 24, 20, 16, 16, 16, 16              db  0,  0,  0,  0,  0,  0,  0,  0,  8,  0,  8,  0z_filter_s:   db  0,  0,  0,  1,  1,  2,  2,  3,  3,  4,  4,  5,  5,  6,  6,  7              db  7,  8,  8,  9,  9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15              db 15, 15, 15, 15, 15, 15, 15, 15 ; should be in one cache linepb_128:       times 4 db 128 ; those are just placed here for alignment.pb_36_m4:     times 2 db 36, -4z3_shuf:      db  8,  7,  7,  6,  6,  5,  5,  4,  4,  3,  3,  2,  2,  1,  1,  0z_filter_t0:  db 55,127, 39,127, 39,127,  7, 15, 31,  7, 15, 31,  0,  3, 31,  0z_filter_t1:  db 39, 63, 19, 47, 19, 47,  3,  3,  3,  3,  3,  3,  0,  0,  0,  0z_upsample1:  db  1,  0,  2,  1,  3,  2,  4,  3,  5,  4,  6,  5,  7,  6,  8,  7z_upsample2:  db  2,  3,  3,  4,  4,  5,  5,  6,  6,  7,  7,  8,  8,  8,  8,  8z2_upsample:  db  7,  6, 15, 14,  5,  4, 13, 12,  3,  2, 11, 10,  1,  0,  9,  8z1_shuf_w4:   db  0,  1,  1,  2,  2,  3,  3,  4,  8,  9,  9, 10, 10, 11, 11, 12z2_shuf_h2:   db  3,  2,  7,  6, 11, 10, 15, 14,  2,  1,  6,  5, 10,  9, 14, 13z2_shuf_h4:   db  7,  6, 15, 14,  6,  5, 14, 13,  5,  4, 13, 12,  4,  3, 12, 11z3_shuf_w4:   db  4,  3,  3,  2,  2,  1,  1,  0, 12, 11, 11, 10, 10,  9,  9,  8z_transpose4: db  0,  4,  8, 12,  1,  5,  9, 13,  2,  6, 10, 14,  3,  7, 11, 15z_base_inc:   dw   0*64,   1*64,   2*64,   3*64,   4*64,   5*64,   6*64,   7*64              dw  16*64,  17*64,  18*64,  19*64,  20*64,  21*64,  22*64,  23*64z2_base_inc:  dw   1*64,   2*64,   3*64,   4*64,   5*64,   6*64,   7*64,   8*64              dw   9*64,  10*64,  11*64,  12*64,  13*64,  14*64,  15*64,  16*64z2_ymul:      dw  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16z2_y_shuf_h4: db 90, 90, 90, 90, 14, 14, 14, 14, 27, 27, 27, 27, 31, 31, 31, 31 ; 2, 6, 3, 7              db 32, 32, 32, 32, 12, 12, 12, 12,  1,  0,  1,  0,  5, -1, -1, -1 ; 0, 4, 1, 5; vpermd indices in bits 4..6 of filter_shuf1: 0, 2, 6, 4, 1, 3, 7, 5filter_shuf1: db 10,  4, 10,  4, 37,  6,  5,  6,103,  9,  7,  9, 72, -1,  8, -1              db 16,  4,  0,  4, 53,  6,  5,  6,119, 11,  7, 11, 95, -1, 15, -1filter_shuf2: db  3,  4,  3,  4,  5,  6,  5,  6,  7,  2,  7,  2,  1, -1,  1, -1filter_shuf3: db  3,  4,  3,  4,  5,  6,  5,  6,  7, 11,  7, 11; 15, -1, 15, -1pb_127_m127:  times 2 db 127, -127ipred_v_shuf: db  0,  1,  0,  1,  4,  5,  4,  5,  8,  9,  8,  9, 12, 13, 12, 13              db  2,  3,  2,  3,  6,  7,  6,  7, 10, 11, 10, 11, 14, 15, 14, 15ipred_h_shuf: db  7,  7,  7,  7,  3,  3,  3,  3,  5,  5,  5,  5,  1,  1,  1,  1              db  6,  6,  6,  6,  2,  2,  2,  2,  4,  4,  4,  4;  0,  0,  0,  0pw_64:        times 2 dw 64 cfl_ac_444_w16_pad1_shuffle: db 0, -1, 1, -1, 2, -1, 3, -1, 4, -1, 5, -1, 6, -1                             times 9 db 7, -1cfl_ac_w16_pad_shuffle: ; w=16, w_pad=1                        db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15                        ; w=8, w_pad=1 as well as second half of previous onecfl_ac_w8_pad1_shuffle: db 0, 1, 2, 3, 4, 5                        times 5 db 6, 7                        ; w=16,w_pad=2                        db 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15                        times 8 db 14, 15                        ; w=16,w_pad=3                        db 0, 1, 2, 3, 4, 5                        times 13 db 6, 7pb_15to0:               db 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 %define pb_0to15 cfl_ac_w16_pad_shuffle%define pb_1  (ipred_h_shuf+12)%define pb_2  (ipred_h_shuf+20)%define pb_3  (ipred_h_shuf+ 4)%define pb_4  (ipred_h_shuf+24)%define pb_5  (ipred_h_shuf+ 8)%define pb_7  (ipred_h_shuf+ 0)%define pb_8  (z_upsample2 +12)%define pb_12 (z2_y_shuf_h4+20)%define pb_14 (z2_y_shuf_h4+ 4)%define pb_15 (z_filter_s  +32)%define pb_27 (z2_y_shuf_h4+ 8)%define pb_31 (z2_y_shuf_h4+12)%define pb_32 (z2_y_shuf_h4+16)%define pb_90 (z2_y_shuf_h4+ 0)%define pw_1  (z2_y_shuf_h4+24)%define pw_8  (z_filter_k  +32) pw_62:    times 2 dw 62pw_128:   times 2 dw 128pw_255:   times 2 dw 255pw_512:   times 2 dw 512 %macro JMP_TABLE 3-*    %xdefine %1_%2_table (%%table - 2*4)    %xdefine %%base mangle(private_prefix %+ _%1_%2)    %%table:    %rep %0 - 2        dd %%base %+ .%3 - (%%table - 2*4)        %rotate 1    %endrep%endmacro %define ipred_dc_splat_avx2_table (ipred_dc_avx2_table + 10*4)%define ipred_cfl_splat_avx2_table (ipred_cfl_avx2_table + 8*4) JMP_TABLE ipred_smooth,     avx2, w4, w8, w16, w32, w64JMP_TABLE ipred_smooth_v,   avx2, w4, w8, w16, w32, w64JMP_TABLE ipred_smooth_h,   avx2, w4, w8, w16, w32, w64JMP_TABLE ipred_paeth,      avx2, w4, w8, w16, w32, w64JMP_TABLE ipred_filter,     avx2, w4, w8, w16, w32JMP_TABLE ipred_dc,         avx2, h4, h8, h16, h32, h64, w4, w8, w16, w32, w64, \                                  s4-10*4, s8-10*4, s16-10*4, s32-10*4, s64-10*4JMP_TABLE ipred_dc_left,    avx2, h4, h8, h16, h32, h64JMP_TABLE ipred_h,          avx2, w4, w8, w16, w32, w64JMP_TABLE ipred_z1,         avx2, w4, w8, w16, w32, w64JMP_TABLE ipred_z2,         avx2, w4, w8, w16, w32, w64JMP_TABLE ipred_z3,         avx2, h4, h8, h16, h32, h64JMP_TABLE ipred_cfl,        avx2, h4, h8, h16, h32, w4, w8, w16, w32, \                                  s4-8*4, s8-8*4, s16-8*4, s32-8*4JMP_TABLE ipred_cfl_left,   avx2, h4, h8, h16, h32JMP_TABLE ipred_cfl_ac_420, avx2, w16_pad1, w16_pad2, w16_pad3JMP_TABLE ipred_cfl_ac_422, avx2, w16_pad1, w16_pad2, w16_pad3JMP_TABLE ipred_cfl_ac_444, avx2, w32_pad1, w32_pad2, w32_pad3, w4, w8, w16, w32JMP_TABLE pal_pred,         avx2, w4, w8, w16, w32, w64 cextern dr_intra_derivativecextern filter_intra_taps SECTION .text INIT_YMM avx2cglobal ipred_dc_top, 3, 7, 6, dst, stride, tl, w, h    lea                  r5, [ipred_dc_left_avx2_table]    tzcnt                wd, wm    inc                 tlq    movu                 m0, [tlq]    movifnidn            hd, hm    mov                 r6d, 0x8000    shrx                r6d, r6d, wd    movd                xm3, r6d    movsxd               r6, [r5+wq*4]    pcmpeqd              m2, m2    pmaddubsw            m0, m2    add                  r6, r5    add                  r5, ipred_dc_splat_avx2_table-ipred_dc_left_avx2_table    movsxd               wq, [r5+wq*4]    add                  wq, r5    jmp                  r6 cglobal ipred_dc_left, 3, 7, 6, dst, stride, tl, w, h, stride3    mov                  hd, hm ; zero upper half    tzcnt               r6d, hd    sub                 tlq, hq    tzcnt                wd, wm    movu                 m0, [tlq]    mov                 r5d, 0x8000    shrx                r5d, r5d, r6d    movd                xm3, r5d    lea                  r5, [ipred_dc_left_avx2_table]    movsxd               r6, [r5+r6*4]    pcmpeqd              m2, m2    pmaddubsw            m0, m2    add                  r6, r5    add                  r5, ipred_dc_splat_avx2_table-ipred_dc_left_avx2_table    movsxd               wq, [r5+wq*4]    add                  wq, r5    jmp                  r6.h64:    movu                 m1, [tlq+32] ; unaligned when jumping here from dc_top    pmaddubsw            m1, m2    paddw                m0, m1.h32:    vextracti128        xm1, m0, 1    paddw               xm0, xm1.h16:    punpckhqdq          xm1, xm0, xm0    paddw               xm0, xm1.h8:    psrlq               xm1, xm0, 32    paddw               xm0, xm1.h4:    pmaddwd             xm0, xm2    pmulhrsw            xm0, xm3    lea            stride3q, [strideq*3]    vpbroadcastb         m0, xm0    mova                 m1, m0    jmp                  wq cglobal ipred_dc, 3, 7, 6, dst, stride, tl, w, h, stride3    movifnidn            hd, hm    movifnidn            wd, wm    tzcnt               r6d, hd    lea                 r5d, [wq+hq]    movd                xm4, r5d    tzcnt               r5d, r5d    movd                xm5, r5d    lea                  r5, [ipred_dc_avx2_table]    tzcnt                wd, wd    movsxd               r6, [r5+r6*4]    movsxd               wq, [r5+wq*4+5*4]    pcmpeqd              m3, m3    psrlw               xm4, 1    add                  r6, r5    add                  wq, r5    lea            stride3q, [strideq*3]    jmp                  r6.h4:    movd                xm0, [tlq-4]    pmaddubsw           xm0, xm3    jmp                  wq.w4:    movd                xm1, [tlq+1]    pmaddubsw           xm1, xm3    psubw               xm0, xm4    paddw               xm0, xm1    pmaddwd             xm0, xm3    cmp                  hd, 4    jg .w4_mul    psrlw               xm0, 3    jmp .w4_end.w4_mul:    punpckhqdq          xm1, xm0, xm0    lea                 r2d, [hq*2]    mov                 r6d, 0x55563334    paddw               xm0, xm1    shrx                r6d, r6d, r2d    psrlq               xm1, xm0, 32    paddw               xm0, xm1    movd                xm1, r6d    psrlw               xm0, 2    pmulhuw             xm0, xm1.w4_end:    vpbroadcastb        xm0, xm0.s4:    movd   [dstq+strideq*0], xm0    movd   [dstq+strideq*1], xm0    movd   [dstq+strideq*2], xm0    movd   [dstq+stride3q ], xm0    lea                dstq, [dstq+strideq*4]    sub                  hd, 4    jg .s4    RETALIGN function_align.h8:    movq                xm0, [tlq-8]    pmaddubsw           xm0, xm3    jmp                  wq.w8:    movq                xm1, [tlq+1]    vextracti128        xm2, m0, 1    pmaddubsw           xm1, xm3    psubw               xm0, xm4    paddw               xm0, xm2    punpckhqdq          xm2, xm0, xm0    paddw               xm0, xm2    paddw               xm0, xm1    psrlq               xm1, xm0, 32    paddw               xm0, xm1    pmaddwd             xm0, xm3    psrlw               xm0, xm5    cmp                  hd, 8    je .w8_end    mov                 r6d, 0x5556    mov                 r2d, 0x3334    cmp                  hd, 32    cmove               r6d, r2d    movd                xm1, r6d    pmulhuw             xm0, xm1.w8_end:    vpbroadcastb        xm0, xm0.s8:    movq   [dstq+strideq*0], xm0    movq   [dstq+strideq*1], xm0    movq   [dstq+strideq*2], xm0    movq   [dstq+stride3q ], xm0    lea                dstq, [dstq+strideq*4]    sub                  hd, 4    jg .s8    RETALIGN function_align.h16:    mova                xm0, [tlq-16]    pmaddubsw           xm0, xm3    jmp                  wq.w16:    movu                xm1, [tlq+1]    vextracti128        xm2, m0, 1    pmaddubsw           xm1, xm3    psubw               xm0, xm4    paddw               xm0, xm2    paddw               xm0, xm1    punpckhqdq          xm1, xm0, xm0    paddw               xm0, xm1    psrlq               xm1, xm0, 32    paddw               xm0, xm1    pmaddwd             xm0, xm3    psrlw               xm0, xm5    cmp                  hd, 16    je .w16_end    mov                 r6d, 0x5556    mov                 r2d, 0x3334    test                 hb, 8|32    cmovz               r6d, r2d    movd                xm1, r6d    pmulhuw             xm0, xm1.w16_end:    vpbroadcastb        xm0, xm0.s16:    mova   [dstq+strideq*0], xm0    mova   [dstq+strideq*1], xm0    mova   [dstq+strideq*2], xm0    mova   [dstq+stride3q ], xm0    lea                dstq, [dstq+strideq*4]    sub                  hd, 4    jg .s16    RETALIGN function_align.h32:    mova                 m0, [tlq-32]    pmaddubsw            m0, m3    jmp                  wq.w32:    movu                 m1, [tlq+1]    pmaddubsw            m1, m3    paddw                m0, m1    vextracti128        xm1, m0, 1    psubw               xm0, xm4    paddw               xm0, xm1    punpckhqdq          xm1, xm0, xm0    paddw               xm0, xm1    psrlq               xm1, xm0, 32    paddw               xm0, xm1    pmaddwd             xm0, xm3    psrlw               xm0, xm5    cmp                  hd, 32    je .w32_end    lea                 r2d, [hq*2]    mov                 r6d, 0x33345556    shrx                r6d, r6d, r2d    movd                xm1, r6d    pmulhuw             xm0, xm1.w32_end:    vpbroadcastb         m0, xm0.s32:    mova   [dstq+strideq*0], m0    mova   [dstq+strideq*1], m0    mova   [dstq+strideq*2], m0    mova   [dstq+stride3q ], m0    lea                dstq, [dstq+strideq*4]    sub                  hd, 4    jg .s32    RETALIGN function_align.h64:    mova                 m0, [tlq-64]    mova                 m1, [tlq-32]    pmaddubsw            m0, m3    pmaddubsw            m1, m3    paddw                m0, m1    jmp                  wq.w64:    movu                 m1, [tlq+ 1]    movu                 m2, [tlq+33]    pmaddubsw            m1, m3    pmaddubsw            m2, m3    paddw                m0, m1    paddw                m0, m2    vextracti128        xm1, m0, 1    psubw               xm0, xm4    paddw               xm0, xm1    punpckhqdq          xm1, xm0, xm0    paddw               xm0, xm1    psrlq               xm1, xm0, 32    paddw               xm0, xm1    pmaddwd             xm0, xm3    psrlw               xm0, xm5    cmp                  hd, 64    je .w64_end    mov                 r6d, 0x33345556    shrx                r6d, r6d, hd    movd                xm1, r6d    pmulhuw             xm0, xm1.w64_end:    vpbroadcastb         m0, xm0    mova                 m1, m0.s64:    mova [dstq+strideq*0+32*0], m0    mova [dstq+strideq*0+32*1], m1    mova [dstq+strideq*1+32*0], m0    mova [dstq+strideq*1+32*1], m1    mova [dstq+strideq*2+32*0], m0    mova [dstq+strideq*2+32*1], m1    mova [dstq+stride3q +32*0], m0    mova [dstq+stride3q +32*1], m1    lea                dstq, [dstq+strideq*4]    sub                  hd, 4    jg .s64    RET cglobal ipred_dc_128, 2, 7, 6, dst, stride, tl, w, h, stride3    lea                  r5, [ipred_dc_splat_avx2_table]    tzcnt                wd, wm    movifnidn            hd, hm    movsxd               wq, [r5+wq*4]    vpbroadcastd         m0, [r5-ipred_dc_splat_avx2_table+pb_128]    mova                 m1, m0    add                  wq, r5    lea            stride3q, [strideq*3]    jmp                  wq cglobal ipred_v, 3, 7, 6, dst, stride, tl, w, h, stride3    lea                  r5, [ipred_dc_splat_avx2_table]    tzcnt                wd, wm    movu                 m0, [tlq+ 1]    movu                 m1, [tlq+33]    movifnidn            hd, hm    movsxd               wq, [r5+wq*4]    add                  wq, r5    lea            stride3q, [strideq*3]    jmp                  wq %macro IPRED_H 2 ; w, store_type    vpbroadcastb         m0, [tlq-1]    vpbroadcastb         m1, [tlq-2]    vpbroadcastb         m2, [tlq-3]    sub                 tlq, 4    vpbroadcastb         m3, [tlq+0]    mov%2  [dstq+strideq*0], m0    mov%2  [dstq+strideq*1], m1    mov%2  [dstq+strideq*2], m2    mov%2  [dstq+stride3q ], m3    lea                dstq, [dstq+strideq*4]    sub                  hd, 4    jg .w%1    RETALIGN function_align%endmacro INIT_XMM avx2cglobal ipred_h, 3, 6, 4, dst, stride, tl, w, h, stride3    lea                  r5, [ipred_h_avx2_table]    tzcnt                wd, wm    movifnidn            hd, hm    movsxd               wq, [r5+wq*4]    add                  wq, r5    lea            stride3q, [strideq*3]    jmp                  wq.w4:    IPRED_H               4, d.w8:    IPRED_H               8, q

Lines 1–500 of 5387. Load more