History

x86: Update x86inc.asm

Make things up-to-date with upstream. https://code.videolan.org/videolan/x86inc.asm

Henrik Gramner committed 2 years ago

Commit afa471d1 parent 782c4df12 files · +519 −270Browse files

Unified · Split

12 files
  1. configure +0 −4
  2. libavcodec/x86/celt_pvq_search.asm +2 −2
  3. libavcodec/x86/h264_chromamc.asm +5 −5
  4. libavcodec/x86/h264_idct.asm +3 −3
  5. libavcodec/x86/h264_intrapred.asm +6 −9
  6. libavcodec/x86/hevc_mc.asm +8 −8
  7. libavcodec/x86/rv40dsp.asm +6 −6
  8. libavcodec/x86/sbrdsp.asm +3 −3
  9. libavcodec/x86/vp8dsp.asm +15 −15
  10. libavcodec/x86/vp9itxfm.asm +2 −0
  11. libavcodec/x86/vp9itxfm_16bpp.asm +7 −5
  12. libavutil/x86/x86inc.asm +462 −210
configure+0−4
@@ -2222,7 +2222,6 @@ ARCH_EXT_LIST_RISCV=" · 2221 unmodified lines
2222 2222
2223ARCH_EXT_LIST_X86="2223ARCH_EXT_LIST_X86="
2224 $ARCH_EXT_LIST_X86_SIMD2224 $ARCH_EXT_LIST_X86_SIMD
2225 cpunop
2226 i6862225 i686
2227"2226"
2228 2227
@@ -2771,7 +2770,6 @@ mipsdsp_deps="mips" · 542 unmodified lines
2771mipsdspr2_deps="mips"2770mipsdspr2_deps="mips"
2772msa_deps="mipsfpu"2771msa_deps="mipsfpu"
2773 2772
2774cpunop_deps="i686"
2775x86_64_select="i686"2773x86_64_select="i686"
2776x86_64_suggest="fast_cmov"2774x86_64_suggest="fast_cmov"
2777 2775
@@ -6401,7 +6399,6 @@ EOF · 3623 unmodified lines
6401 done6399 done
6402 disabled x86asm && die "nasm/yasm not found or too old. Use --disable-x86asm for a crippled build."6400 disabled x86asm && die "nasm/yasm not found or too old. Use --disable-x86asm for a crippled build."
6403 X86ASMFLAGS="-f $objformat"6401 X86ASMFLAGS="-f $objformat"
6404 enabled pic && append X86ASMFLAGS "-DPIC"
6405 test -n "$extern_prefix" && append X86ASMFLAGS "-DPREFIX"6402 test -n "$extern_prefix" && append X86ASMFLAGS "-DPREFIX"
6406 case "$objformat" in6403 case "$objformat" in
6407 elf*) enabled debug && append X86ASMFLAGS $x86asm_debug ;;6404 elf*) enabled debug && append X86ASMFLAGS $x86asm_debug ;;
@@ -6412,7 +6409,6 @@ EOF · 4 unmodified lines
6412 enabled avx2 && check_x86asm avx2_external "vextracti128 xmm0, ymm0, 0"6409 enabled avx2 && check_x86asm avx2_external "vextracti128 xmm0, ymm0, 0"
6413 enabled xop && check_x86asm xop_external "vpmacsdd xmm0, xmm1, xmm2, xmm3"6410 enabled xop && check_x86asm xop_external "vpmacsdd xmm0, xmm1, xmm2, xmm3"
6414 enabled fma4 && check_x86asm fma4_external "vfmaddps ymm0, ymm1, ymm2, ymm3"6411 enabled fma4 && check_x86asm fma4_external "vfmaddps ymm0, ymm1, ymm2, ymm3"
6415 check_x86asm cpunop "CPU amdnop"
6416 fi6412 fi
6417 6413
6418 case "$cpu" in6414 case "$cpu" in
libavcodec/x86/celt_pvq_search.asm+2−2
@@ -74,7 +74,7 @@ SECTION .text · 73 unmodified lines
74; "movaps m0, [r5 + r4]" if PIC is enabled74; "movaps m0, [r5 + r4]" if PIC is enabled
75; "movaps m0, [constant_name + r4]" if texrel are used75; "movaps m0, [constant_name + r4]" if texrel are used
76%macro SET_PIC_BASE 3; reg, const_label76%macro SET_PIC_BASE 3; reg, const_label
77%ifdef PIC77%if PIC
78 %{1} %2, [%3] ; lea r5, [rip+const]78 %{1} %2, [%3] ; lea r5, [rip+const]
79 %define pic_base_%3 %279 %define pic_base_%3 %2
80%else80%else
@@ -195,7 +195,7 @@ align 16 · 114 unmodified lines
195; PIC relative addressing. Use this195; PIC relative addressing. Use this
196; to count it in cglobal196; to count it in cglobal
197;197;
198%ifdef PIC198%if PIC
199 %define num_pic_regs 1199 %define num_pic_regs 1
200%else200%else
201 %define num_pic_regs 0201 %define num_pic_regs 0
libavcodec/x86/h264_chromamc.asm+5−5
@@ -91,7 +91,7 @@ SECTION .text · 90 unmodified lines
91 91
92%macro chroma_mc8_mmx_func 2-392%macro chroma_mc8_mmx_func 2-3
93%ifidn %2, rv4093%ifidn %2, rv40
94%ifdef PIC94%if PIC
95%define rnd_1d_rv40 r895%define rnd_1d_rv40 r8
96%define rnd_2d_rv40 r896%define rnd_2d_rv40 r8
97%define extra_regs 297%define extra_regs 2
@@ -147,7 +147,7 @@ cglobal %1_%2_chroma_mc8%3, 6, 7 + extra_regs, 0 · 49 unmodified lines
147 or r4d, r5d ; x + y147 or r4d, r5d ; x + y
148 148
149%ifidn %2, rv40149%ifidn %2, rv40
150%ifdef PIC150%if PIC
151 lea r8, [rnd_rv40_1d_tbl]151 lea r8, [rnd_rv40_1d_tbl]
152%endif152%endif
153%if ARCH_X86_64 == 0153%if ARCH_X86_64 == 0
@@ -198,7 +198,7 @@ cglobal %1_%2_chroma_mc8%3, 6, 7 + extra_regs, 0 · 44 unmodified lines
198 movd m4, r4d ; x198 movd m4, r4d ; x
199 movd m6, r5d ; y199 movd m6, r5d ; y
200%ifidn %2, rv40200%ifidn %2, rv40
201%ifdef PIC201%if PIC
202 lea r8, [rnd_rv40_2d_tbl]202 lea r8, [rnd_rv40_2d_tbl]
203%endif203%endif
204%if ARCH_X86_64 == 0204%if ARCH_X86_64 == 0
@@ -283,7 +283,7 @@ cglobal %1_%2_chroma_mc8%3, 6, 7 + extra_regs, 0 · 78 unmodified lines
283%macro chroma_mc4_mmx_func 2283%macro chroma_mc4_mmx_func 2
284%define extra_regs 0284%define extra_regs 0
285%ifidn %2, rv40285%ifidn %2, rv40
286%ifdef PIC286%if PIC
287%define extra_regs 1287%define extra_regs 1
288%endif ; PIC288%endif ; PIC
289%endif ; rv40289%endif ; rv40
@@ -301,7 +301,7 @@ cglobal %1_%2_chroma_mc4, 6, 6 + extra_regs, 0 · 11 unmodified lines
301 psubw m5, m3301 psubw m5, m3
302 302
303%ifidn %2, rv40303%ifidn %2, rv40
304%ifdef PIC304%if PIC
305 lea r6, [rnd_rv40_2d_tbl]305 lea r6, [rnd_rv40_2d_tbl]
306%define rnd_2d_rv40 r6306%define rnd_2d_rv40 r6
307%else307%else
libavcodec/x86/h264_idct.asm+3−3
@@ -42,7 +42,7 @@ scan8_mem: db 4+ 1*8, 5+ 1*8, 4+ 2*8, 5+ 2*8 · 41 unmodified lines
42 db 6+11*8, 7+11*8, 6+12*8, 7+12*842 db 6+11*8, 7+11*8, 6+12*8, 7+12*8
43 db 4+13*8, 5+13*8, 4+14*8, 5+14*843 db 4+13*8, 5+13*8, 4+14*8, 5+14*8
44 db 6+13*8, 7+13*8, 6+14*8, 7+14*844 db 6+13*8, 7+13*8, 6+14*8, 7+14*8
45%ifdef PIC45%if PIC
46%define npicregs 146%define npicregs 1
47%define scan8 picregq47%define scan8 picregq
48%else48%else
@@ -322,7 +322,7 @@ INIT_XMM sse2 · 273 unmodified lines
322cglobal h264_idct8_add4_8, 5, 8 + npicregs, 10, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg322cglobal h264_idct8_add4_8, 5, 8 + npicregs, 10, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
323 movsxdifnidn r3, r3d323 movsxdifnidn r3, r3d
324 xor r5, r5324 xor r5, r5
325%ifdef PIC325%if PIC
326 lea picregq, [scan8_mem]326 lea picregq, [scan8_mem]
327%endif327%endif
328.nextblock:328.nextblock:
@@ -398,7 +398,7 @@ h264_idct_add8_mmx_plane: · 69 unmodified lines
398cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg398cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
399; dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg399; dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
400 movsxdifnidn r3, r3d400 movsxdifnidn r3, r3d
401%ifdef PIC401%if PIC
402 lea picregq, [scan8_mem]402 lea picregq, [scan8_mem]
403%endif403%endif
404%if ARCH_X86_64404%if ARCH_X86_64
libavcodec/x86/h264_intrapred.asm+6−9
@@ -1311,10 +1311,7 @@ PRED8x8L_DOWN_RIGHT · 1310 unmodified lines
1311;-----------------------------------------------------------------------------1311;-----------------------------------------------------------------------------
1312 1312
1313%macro PRED8x8L_VERTICAL_RIGHT 01313%macro PRED8x8L_VERTICAL_RIGHT 0
1314cglobal pred8x8l_vertical_right_8, 4,5,71314cglobal pred8x8l_vertical_right_8, 4,5,6
1315 ; manually spill XMM registers for Win64 because
1316 ; the code here is initialized with INIT_MMX
1317 WIN64_SPILL_XMM 7
1318 sub r0, r31315 sub r0, r3
1319 lea r4, [r0+r3*2]1316 lea r4, [r0+r3*2]
1320 movq mm0, [r0+r3*1-8]1317 movq mm0, [r0+r3*1-8]
@@ -1384,7 +1381,6 @@ cglobal pred8x8l_vertical_right_8, 4,5,7 · 63 unmodified lines
1384 movq2dq xmm4, mm61381 movq2dq xmm4, mm6
1385 pslldq xmm4, 81382 pslldq xmm4, 8
1386 por xmm0, xmm41383 por xmm0, xmm4
1387 movdqa xmm6, [pw_ff00]
1388 movdqa xmm1, xmm01384 movdqa xmm1, xmm0
1389 lea r2, [r1+r3*2]1385 lea r2, [r1+r3*2]
1390 movdqa xmm2, xmm01386 movdqa xmm2, xmm0
@@ -1394,15 +1390,16 @@ cglobal pred8x8l_vertical_right_8, 4,5,7 · 3 unmodified lines
1394 pavgb xmm2, xmm01390 pavgb xmm2, xmm0
1395INIT_XMM cpuname1391INIT_XMM cpuname
1396 PRED4x4_LOWPASS xmm4, xmm3, xmm1, xmm0, xmm51392 PRED4x4_LOWPASS xmm4, xmm3, xmm1, xmm0, xmm5
1397 pandn xmm6, xmm41393 movdqa xmm0, [pw_ff00]
1394 pandn xmm0, xmm4
1398 movdqa xmm5, xmm41395 movdqa xmm5, xmm4
1399 psrlw xmm4, 81396 psrlw xmm4, 8
1400 packuswb xmm6, xmm41397 packuswb xmm0, xmm4
1401 movhlps xmm4, xmm61398 movhlps xmm4, xmm0
1402 movhps [r0+r3*2], xmm51399 movhps [r0+r3*2], xmm5
1403 movhps [r0+r3*1], xmm21400 movhps [r0+r3*1], xmm2
1404 psrldq xmm5, 41401 psrldq xmm5, 4
1405 movss xmm5, xmm61402 movss xmm5, xmm0
1406 psrldq xmm2, 41403 psrldq xmm2, 4
1407 movss xmm2, xmm41404 movss xmm2, xmm4
1408 lea r0, [r2+r3*2]1405 lea r0, [r2+r3*2]
libavcodec/x86/hevc_mc.asm+8−8
@@ -180,7 +180,7 @@ SECTION .text · 179 unmodified lines
180%macro EPEL_FILTER 5 ; bit depth, filter index, xmma, xmmb, gprtmp180%macro EPEL_FILTER 5 ; bit depth, filter index, xmma, xmmb, gprtmp
181%if cpuflag(avx2)181%if cpuflag(avx2)
182%assign %%offset 32182%assign %%offset 32
183%ifdef PIC183%if PIC
184 lea %5q, [hevc_epel_filters_avx2_%1]184 lea %5q, [hevc_epel_filters_avx2_%1]
185 %define FILTER %5q185 %define FILTER %5q
186%else186%else
@@ -188,7 +188,7 @@ SECTION .text · 1 unmodified line
188%endif188%endif
189%else189%else
190%assign %%offset 16190%assign %%offset 16
191%ifdef PIC191%if PIC
192 lea %5q, [hevc_epel_filters_sse4_%1]192 lea %5q, [hevc_epel_filters_sse4_%1]
193 %define FILTER %5q193 %define FILTER %5q
194%else194%else
@@ -216,7 +216,7 @@ SECTION .text · 21 unmodified lines
216%define %%table hevc_epel_filters_sse4_%1216%define %%table hevc_epel_filters_sse4_%1
217%endif217%endif
218 218
219%ifdef PIC219%if PIC
220 lea r3srcq, [%%table]220 lea r3srcq, [%%table]
221 %define FILTER r3srcq221 %define FILTER r3srcq
222%else222%else
@@ -234,7 +234,7 @@ SECTION .text · 11 unmodified lines
234%else234%else
235%define %%table hevc_epel_filters_sse4_10235%define %%table hevc_epel_filters_sse4_10
236%endif236%endif
237%ifdef PIC237%if PIC
238 lea r3srcq, [%%table]238 lea r3srcq, [%%table]
239 %define FILTER r3srcq239 %define FILTER r3srcq
240%else240%else
@@ -257,7 +257,7 @@ SECTION .text · 16 unmodified lines
257%define %%table hevc_qpel_filters_sse4_%1257%define %%table hevc_qpel_filters_sse4_%1
258%endif258%endif
259 259
260%ifdef PIC260%if PIC
261 lea rfilterq, [%%table]261 lea rfilterq, [%%table]
262%else262%else
263 %define rfilterq %%table263 %define rfilterq %%table
@@ -576,7 +576,7 @@ SECTION .text · 312 unmodified lines
576%define %%table hevc_qpel_filters_sse4_%2576%define %%table hevc_qpel_filters_sse4_%2
577%endif577%endif
578 578
579%ifdef PIC579%if PIC
580 lea rfilterq, [%%table]580 lea rfilterq, [%%table]
581%else581%else
582 %define rfilterq %%table582 %define rfilterq %%table
@@ -1288,7 +1288,7 @@ HEVC_PUT_HEVC_QPEL_HV 16, 10 · 705 unmodified lines
1288%assign %%offset 41288%assign %%offset 4
1289 dec %2q1289 dec %2q
1290 shl %2q, 31290 shl %2q, 3
1291%ifdef PIC1291%if PIC
1292 lea %5q, [%%table]1292 lea %5q, [%%table]
1293 %define FILTER %5q1293 %define FILTER %5q
1294%else1294%else
@@ -1365,7 +1365,7 @@ cglobal hevc_put_hevc_qpel_hv%1_%2, 6, 7, 27, dst, src, srcstride, height, mx, m · 70 unmodified lines
1365 sub myq, 11365 sub myq, 1
1366 shl myq, 51366 shl myq, 5
1367%define %%table hevc_qpel_filters_avx512icl_v_%11367%define %%table hevc_qpel_filters_avx512icl_v_%1
1368%ifdef PIC1368%if PIC
1369 lea tmpq, [%%table]1369 lea tmpq, [%%table]
1370 %define FILTER tmpq1370 %define FILTER tmpq
1371%else1371%else
libavcodec/x86/rv40dsp.asm+6−6
@@ -51,7 +51,7 @@ sixtap_filter_v_m: times 8 dw 1 · 50 unmodified lines
51 times 8 dw 2051 times 8 dw 20
52 times 8 dw 5252 times 8 dw 52
53 53
54%ifdef PIC54%if PIC
55%define sixtap_filter_hw picregq55%define sixtap_filter_hw picregq
56%define sixtap_filter_hb picregq56%define sixtap_filter_hb picregq
57%define sixtap_filter_v picregq57%define sixtap_filter_v picregq
@@ -84,7 +84,7 @@ SECTION .text · 26 unmodified lines
84%if WIN6484%if WIN64
85 movsxd %1q, %1d85 movsxd %1q, %1d
86%endif86%endif
87%ifdef PIC87%if PIC
88 add %1q, picregq88 add %1q, picregq
89%else89%else
90 add %1q, %290 add %1q, %2
@@ -104,7 +104,7 @@ SECTION .text · 13 unmodified lines
104 104
105%macro FILTER_V 1105%macro FILTER_V 1
106cglobal %1_rv40_qpel_v, 6,6+npicregs,12, dst, dststride, src, srcstride, height, my, picreg106cglobal %1_rv40_qpel_v, 6,6+npicregs,12, dst, dststride, src, srcstride, height, my, picreg
107%ifdef PIC107%if PIC
108 lea picregq, [sixtap_filter_v_m]108 lea picregq, [sixtap_filter_v_m]
109%endif109%endif
110 pxor m7, m7110 pxor m7, m7
@@ -175,7 +175,7 @@ cglobal %1_rv40_qpel_v, 6,6+npicregs,12, dst, dststride, src, srcstride, height, · 64 unmodified lines
175 175
176%macro FILTER_H 1176%macro FILTER_H 1
177cglobal %1_rv40_qpel_h, 6, 6+npicregs, 12, dst, dststride, src, srcstride, height, mx, picreg177cglobal %1_rv40_qpel_h, 6, 6+npicregs, 12, dst, dststride, src, srcstride, height, mx, picreg
178%ifdef PIC178%if PIC
179 lea picregq, [sixtap_filter_v_m]179 lea picregq, [sixtap_filter_v_m]
180%endif180%endif
181 pxor m7, m7181 pxor m7, m7
@@ -238,7 +238,7 @@ FILTER_V avg · 56 unmodified lines
238 238
239%macro FILTER_SSSE3 1239%macro FILTER_SSSE3 1
240cglobal %1_rv40_qpel_v, 6,6+npicregs,8, dst, dststride, src, srcstride, height, my, picreg240cglobal %1_rv40_qpel_v, 6,6+npicregs,8, dst, dststride, src, srcstride, height, my, picreg
241%ifdef PIC241%if PIC
242 lea picregq, [sixtap_filter_hb_m]242 lea picregq, [sixtap_filter_hb_m]
243%endif243%endif
244 244
@@ -283,7 +283,7 @@ cglobal %1_rv40_qpel_v, 6,6+npicregs,8, dst, dststride, src, srcstride, height, · 38 unmodified lines
283 RET283 RET
284 284
285cglobal %1_rv40_qpel_h, 6,6+npicregs,8, dst, dststride, src, srcstride, height, mx, picreg285cglobal %1_rv40_qpel_h, 6,6+npicregs,8, dst, dststride, src, srcstride, height, mx, picreg
286%ifdef PIC286%if PIC
287 lea picregq, [sixtap_filter_hb_m]287 lea picregq, [sixtap_filter_hb_m]
288%endif288%endif
289 mova m3, [filter_h6_shuf2]289 mova m3, [filter_h6_shuf2]
libavcodec/x86/sbrdsp.asm+3−3
@@ -308,7 +308,7 @@ cglobal sbr_qmf_pre_shuffle, 1,4,6,z · 307 unmodified lines
308 movq [r2q], m2308 movq [r2q], m2
309 RET309 RET
310 310
311%ifdef PIC311%if PIC
312%define NREGS 1312%define NREGS 1
313%if UNIX64313%if UNIX64
314%define NOISE_TABLE r6q ; r5q is m_max314%define NOISE_TABLE r6q ; r5q is m_max
@@ -321,7 +321,7 @@ cglobal sbr_qmf_pre_shuffle, 1,4,6,z · 6 unmodified lines
321%endif321%endif
322 322
323%macro LOAD_NST 1323%macro LOAD_NST 1
324%ifdef PIC324%if PIC
325 lea NOISE_TABLE, [%1]325 lea NOISE_TABLE, [%1]
326 mova m0, [kxq + NOISE_TABLE]326 mova m0, [kxq + NOISE_TABLE]
327%else327%else
@@ -371,7 +371,7 @@ apply_noise_main: · 43 unmodified lines
371 movsxdifnidn noiseq, noised371 movsxdifnidn noiseq, noised
372 dec noiseq372 dec noiseq
373 shl countd, 2373 shl countd, 2
374%ifdef PIC374%if PIC
375 lea NOISE_TABLE, [sbr_noise_table]375 lea NOISE_TABLE, [sbr_noise_table]
376%endif376%endif
377 lea Yq, [Yq + 2*countq]377 lea Yq, [Yq + 2*countq]
libavcodec/x86/vp8dsp.asm+15−15
@@ -114,7 +114,7 @@ bilinear_filter_vb_m: times 8 db 7, 1 · 113 unmodified lines
114 times 8 db 2, 6114 times 8 db 2, 6
115 times 8 db 1, 7115 times 8 db 1, 7
116 116
117%ifdef PIC117%if PIC
118%define fourtap_filter_hw picregq118%define fourtap_filter_hw picregq
119%define sixtap_filter_hw picregq119%define sixtap_filter_hw picregq
120%define fourtap_filter_hb picregq120%define fourtap_filter_hb picregq
@@ -166,7 +166,7 @@ cglobal put_vp8_epel%1_h6, 6, 6 + npicregs, 8, dst, dststride, src, srcstride, h · 45 unmodified lines
166 lea mxd, [mxq*3]166 lea mxd, [mxq*3]
167 mova m3, [filter_h6_shuf2]167 mova m3, [filter_h6_shuf2]
168 mova m4, [filter_h6_shuf3]168 mova m4, [filter_h6_shuf3]
169%ifdef PIC169%if PIC
170 lea picregq, [sixtap_filter_hb_m]170 lea picregq, [sixtap_filter_hb_m]
171%endif171%endif
172 mova m5, [sixtap_filter_hb+mxq*8-48] ; set up 6tap filter in bytes172 mova m5, [sixtap_filter_hb+mxq*8-48] ; set up 6tap filter in bytes
@@ -207,7 +207,7 @@ cglobal put_vp8_epel%1_h4, 6, 6 + npicregs, 7, dst, dststride, src, srcstride, h · 34 unmodified lines
207 mova m2, [pw_256]207 mova m2, [pw_256]
208 mova m3, [filter_h2_shuf]208 mova m3, [filter_h2_shuf]
209 mova m4, [filter_h4_shuf]209 mova m4, [filter_h4_shuf]
210%ifdef PIC210%if PIC
211 lea picregq, [fourtap_filter_hb_m]211 lea picregq, [fourtap_filter_hb_m]
212%endif212%endif
213 mova m5, [fourtap_filter_hb+mxq-16] ; set up 4tap filter in bytes213 mova m5, [fourtap_filter_hb+mxq-16] ; set up 4tap filter in bytes
@@ -234,7 +234,7 @@ cglobal put_vp8_epel%1_h4, 6, 6 + npicregs, 7, dst, dststride, src, srcstride, h · 20 unmodified lines
234 234
235cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my235cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my
236 shl myd, 4236 shl myd, 4
237%ifdef PIC237%if PIC
238 lea picregq, [fourtap_filter_hb_m]238 lea picregq, [fourtap_filter_hb_m]
239%endif239%endif
240 mova m5, [fourtap_filter_hb+myq-16]240 mova m5, [fourtap_filter_hb+myq-16]
@@ -272,7 +272,7 @@ cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picr · 31 unmodified lines
272 272
273cglobal put_vp8_epel%1_v6, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my273cglobal put_vp8_epel%1_v6, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my
274 lea myd, [myq*3]274 lea myd, [myq*3]
275%ifdef PIC275%if PIC
276 lea picregq, [sixtap_filter_hb_m]276 lea picregq, [sixtap_filter_hb_m]
277%endif277%endif
278 lea myq, [sixtap_filter_hb+myq*8]278 lea myq, [sixtap_filter_hb+myq*8]
@@ -326,7 +326,7 @@ FILTER_SSSE3 8 · 47 unmodified lines
326INIT_MMX mmxext326INIT_MMX mmxext
327cglobal put_vp8_epel4_h4, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, height, mx, picreg327cglobal put_vp8_epel4_h4, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, height, mx, picreg
328 shl mxd, 4328 shl mxd, 4
329%ifdef PIC329%if PIC
330 lea picregq, [fourtap_filter_hw_m]330 lea picregq, [fourtap_filter_hw_m]
331%endif331%endif
332 movq mm4, [fourtap_filter_hw+mxq-16] ; set up 4tap filter in words332 movq mm4, [fourtap_filter_hw+mxq-16] ; set up 4tap filter in words
@@ -374,7 +374,7 @@ cglobal put_vp8_epel4_h4, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, he · 41 unmodified lines
374INIT_MMX mmxext374INIT_MMX mmxext
375cglobal put_vp8_epel4_h6, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, height, mx, picreg375cglobal put_vp8_epel4_h6, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, height, mx, picreg
376 lea mxd, [mxq*3]376 lea mxd, [mxq*3]
377%ifdef PIC377%if PIC
378 lea picregq, [sixtap_filter_hw_m]378 lea picregq, [sixtap_filter_hw_m]
379%endif379%endif
380 movq mm4, [sixtap_filter_hw+mxq*8-48] ; set up 4tap filter in words380 movq mm4, [sixtap_filter_hw+mxq*8-48] ; set up 4tap filter in words
@@ -431,7 +431,7 @@ cglobal put_vp8_epel4_h6, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, he · 50 unmodified lines
431INIT_XMM sse2431INIT_XMM sse2
432cglobal put_vp8_epel8_h4, 6, 6 + npicregs, 10, dst, dststride, src, srcstride, height, mx, picreg432cglobal put_vp8_epel8_h4, 6, 6 + npicregs, 10, dst, dststride, src, srcstride, height, mx, picreg
433 shl mxd, 5433 shl mxd, 5
434%ifdef PIC434%if PIC
435 lea picregq, [fourtap_filter_v_m]435 lea picregq, [fourtap_filter_v_m]
436%endif436%endif
437 lea mxq, [fourtap_filter_v+mxq-32]437 lea mxq, [fourtap_filter_v+mxq-32]
@@ -480,7 +480,7 @@ INIT_XMM sse2 · 42 unmodified lines
480cglobal put_vp8_epel8_h6, 6, 6 + npicregs, 14, dst, dststride, src, srcstride, height, mx, picreg480cglobal put_vp8_epel8_h6, 6, 6 + npicregs, 14, dst, dststride, src, srcstride, height, mx, picreg
481 lea mxd, [mxq*3]481 lea mxd, [mxq*3]
482 shl mxd, 4482 shl mxd, 4
483%ifdef PIC483%if PIC
484 lea picregq, [sixtap_filter_v_m]484 lea picregq, [sixtap_filter_v_m]
485%endif485%endif
486 lea mxq, [sixtap_filter_v+mxq-96]486 lea mxq, [sixtap_filter_v+mxq-96]
@@ -543,7 +543,7 @@ cglobal put_vp8_epel8_h6, 6, 6 + npicregs, 14, dst, dststride, src, srcstride, h · 56 unmodified lines
543; 4x4 block, V-only 4-tap filter543; 4x4 block, V-only 4-tap filter
544cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my544cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my
545 shl myd, 5545 shl myd, 5
546%ifdef PIC546%if PIC
547 lea picregq, [fourtap_filter_v_m]547 lea picregq, [fourtap_filter_v_m]
548%endif548%endif
549 lea myq, [fourtap_filter_v+myq-32]549 lea myq, [fourtap_filter_v+myq-32]
@@ -597,7 +597,7 @@ cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picr · 47 unmodified lines
597cglobal put_vp8_epel%1_v6, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my597cglobal put_vp8_epel%1_v6, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my
598 shl myd, 4598 shl myd, 4
599 lea myq, [myq*3]599 lea myq, [myq*3]
600%ifdef PIC600%if PIC
601 lea picregq, [sixtap_filter_v_m]601 lea picregq, [sixtap_filter_v_m]
602%endif602%endif
603 lea myq, [sixtap_filter_v+myq-96]603 lea myq, [sixtap_filter_v+myq-96]
@@ -667,7 +667,7 @@ FILTER_V 8 · 63 unmodified lines
667%if cpuflag(ssse3)667%if cpuflag(ssse3)
668cglobal put_vp8_bilinear%1_v, 7, 7, 5, dst, dststride, src, srcstride, height, picreg, my668cglobal put_vp8_bilinear%1_v, 7, 7, 5, dst, dststride, src, srcstride, height, picreg, my
669 shl myd, 4669 shl myd, 4
670%ifdef PIC670%if PIC
671 lea picregq, [bilinear_filter_vb_m]671 lea picregq, [bilinear_filter_vb_m]
672%endif672%endif
673 pxor m4, m4673 pxor m4, m4
@@ -697,7 +697,7 @@ cglobal put_vp8_bilinear%1_v, 7, 7, 5, dst, dststride, src, srcstride, height, p · 23 unmodified lines
697%else ; cpuflag(ssse3)697%else ; cpuflag(ssse3)
698cglobal put_vp8_bilinear%1_v, 7, 7, 7, dst, dststride, src, srcstride, height, picreg, my698cglobal put_vp8_bilinear%1_v, 7, 7, 7, dst, dststride, src, srcstride, height, picreg, my
699 shl myd, 4699 shl myd, 4
700%ifdef PIC700%if PIC
701 lea picregq, [bilinear_filter_vw_m]701 lea picregq, [bilinear_filter_vw_m]
702%endif702%endif
703 pxor m6, m6703 pxor m6, m6
@@ -743,7 +743,7 @@ cglobal put_vp8_bilinear%1_v, 7, 7, 7, dst, dststride, src, srcstride, height, p · 39 unmodified lines
743%if cpuflag(ssse3)743%if cpuflag(ssse3)
744cglobal put_vp8_bilinear%1_h, 6, 6 + npicregs, 5, dst, dststride, src, srcstride, height, mx, picreg744cglobal put_vp8_bilinear%1_h, 6, 6 + npicregs, 5, dst, dststride, src, srcstride, height, mx, picreg
745 shl mxd, 4745 shl mxd, 4
746%ifdef PIC746%if PIC
747 lea picregq, [bilinear_filter_vb_m]747 lea picregq, [bilinear_filter_vb_m]
748%endif748%endif
749 pxor m4, m4749 pxor m4, m4
@@ -773,7 +773,7 @@ cglobal put_vp8_bilinear%1_h, 6, 6 + npicregs, 5, dst, dststride, src, srcstride · 23 unmodified lines
773%else ; cpuflag(ssse3)773%else ; cpuflag(ssse3)
774cglobal put_vp8_bilinear%1_h, 6, 6 + npicregs, 7, dst, dststride, src, srcstride, height, mx, picreg774cglobal put_vp8_bilinear%1_h, 6, 6 + npicregs, 7, dst, dststride, src, srcstride, height, mx, picreg
775 shl mxd, 4775 shl mxd, 4
776%ifdef PIC776%if PIC
777 lea picregq, [bilinear_filter_vw_m]777 lea picregq, [bilinear_filter_vw_m]
778%endif778%endif
779 pxor m6, m6779 pxor m6, m6
libavcodec/x86/vp9itxfm.asm+2−0
@@ -330,7 +330,9 @@ IDCT_4x4_FN ssse3 · 329 unmodified lines
330INIT_MMX %5330INIT_MMX %5
331cglobal vp9_%1_%3_4x4_add, 3, 3, 0, dst, stride, block, eob331cglobal vp9_%1_%3_4x4_add, 3, 3, 0, dst, stride, block, eob
332%if WIN64 && notcpuflag(ssse3)332%if WIN64 && notcpuflag(ssse3)
333INIT_XMM cpuname
333 WIN64_SPILL_XMM 8334 WIN64_SPILL_XMM 8
335INIT_MMX cpuname
334%endif336%endif
335 movdqa xmm5, [pd_8192]337 movdqa xmm5, [pd_8192]
336 mova m0, [blockq+ 0]338 mova m0, [blockq+ 0]
libavcodec/x86/vp9itxfm_16bpp.asm+7−5
@@ -303,7 +303,9 @@ IDCT4_10_FN · 302 unmodified lines
303%macro IADST4_FN 4303%macro IADST4_FN 4
304cglobal vp9_%1_%3_4x4_add_10, 3, 3, 0, dst, stride, block, eob304cglobal vp9_%1_%3_4x4_add_10, 3, 3, 0, dst, stride, block, eob
305%if WIN64 && notcpuflag(ssse3)305%if WIN64 && notcpuflag(ssse3)
306INIT_XMM cpuname
306 WIN64_SPILL_XMM 8307 WIN64_SPILL_XMM 8
308INIT_MMX cpuname
307%endif309%endif
308 movdqa xmm5, [pd_8192]310 movdqa xmm5, [pd_8192]
309 mova m0, [blockq+0*16+0]311 mova m0, [blockq+0*16+0]
@@ -672,7 +674,7 @@ cglobal vp9_idct_idct_8x8_add_10, 4, 6 + ARCH_X86_64, 14, \ · 362 unmodified lines
672 mov dstbakq, dstq674 mov dstbakq, dstq
673 movsxd cntq, cntd675 movsxd cntq, cntd
674%endif676%endif
675%ifdef PIC677%if PIC
676 lea ptrq, [default_8x8]678 lea ptrq, [default_8x8]
677 movzx cntd, byte [ptrq+cntq-1]679 movzx cntd, byte [ptrq+cntq-1]
678%else680%else
@@ -921,7 +923,7 @@ cglobal vp9_%1_%3_8x8_add_10, 4, 6 + ARCH_X86_64, 16, \ · 242 unmodified lines
921 mov dstbakq, dstq923 mov dstbakq, dstq
922 movsxd cntq, cntd924 movsxd cntq, cntd
923%endif925%endif
924%ifdef PIC926%if PIC
925 lea ptrq, [%5_8x8]927 lea ptrq, [%5_8x8]
926 movzx cntd, byte [ptrq+cntq-1]928 movzx cntd, byte [ptrq+cntq-1]
927%else929%else
@@ -1128,7 +1130,7 @@ cglobal vp9_idct_idct_16x16_add_10, 4, 6 + ARCH_X86_64, 16, \ · 200 unmodified lines
1128 mov dstbakq, dstq1130 mov dstbakq, dstq
1129 movsxd cntq, cntd1131 movsxd cntq, cntd
1130%endif1132%endif
1131%ifdef PIC1133%if PIC
1132 lea ptrq, [default_16x16]1134 lea ptrq, [default_16x16]
1133 movzx cntd, byte [ptrq+cntq-1]1135 movzx cntd, byte [ptrq+cntq-1]
1134%else1136%else
@@ -1445,7 +1447,7 @@ cglobal vp9_%1_%4_16x16_add_10, 4, 6 + ARCH_X86_64, 16, \ · 310 unmodified lines
1445 mov dstbakq, dstq1447 mov dstbakq, dstq
1446 movsxd cntq, cntd1448 movsxd cntq, cntd
1447%endif1449%endif
1448%ifdef PIC1450%if PIC
1449 lea ptrq, [%7_16x16]1451 lea ptrq, [%7_16x16]
1450 movzx cntd, byte [ptrq+cntq-1]1452 movzx cntd, byte [ptrq+cntq-1]
1451%else1453%else
@@ -1958,7 +1960,7 @@ cglobal vp9_idct_idct_32x32_add_10, 4, 6 + ARCH_X86_64, 16, \ · 506 unmodified lines
1958 mov dstbakq, dstq1960 mov dstbakq, dstq
1959 movsxd cntq, cntd1961 movsxd cntq, cntd
1960%endif1962%endif
1961%ifdef PIC1963%if PIC
1962 lea ptrq, [default_32x32]1964 lea ptrq, [default_32x32]
1963 movzx cntd, byte [ptrq+cntq-1]1965 movzx cntd, byte [ptrq+cntq-1]
1964%else1966%else
libavutil/x86/x86inc.asm+462−210
@@ -1,7 +1,7 @@
1;*****************************************************************************1;*****************************************************************************
2;* x86inc.asm: x264asm abstraction layer2;* x86inc.asm: x86 abstraction layer
3;*****************************************************************************3;*****************************************************************************
4;* Copyright (C) 2005-2018 x264 project4;* Copyright (C) 2005-2024 x264 project
5;*5;*
6;* Authors: Loren Merritt <lorenm@u.washington.edu>6;* Authors: Loren Merritt <lorenm@u.washington.edu>
7;* Henrik Gramner <henrik@gramner.com>7;* Henrik Gramner <henrik@gramner.com>
@@ -21,21 +21,14 @@ · 13 unmodified lines
21;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.21;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
22;*****************************************************************************22;*****************************************************************************
23 23
24; This is a header file for the x264ASM assembly language, which uses24; This is a header file for the x86inc.asm assembly language, which uses
25; NASM/YASM syntax combined with a large number of macros to provide easy25; NASM/YASM syntax combined with a large number of macros to provide easy
26; abstraction between different calling conventions (x86_32, win64, linux64).26; abstraction between different calling conventions (x86_32, win64, linux64).
27; It also has various other useful features to simplify writing the kind of27; It also has various other useful features to simplify writing the kind of
28; DSP functions that are most often used in x264.28; DSP functions that are most often used.
29
30; Unlike the rest of x264, this file is available under an ISC license, as it
31; has significant usefulness outside of x264 and we want it to be available
32; to the largest audience possible. Of course, if you modify it for your own
33; purposes to add a new feature, we strongly encourage contributing a patch
34; as this feature might be useful for others as well. Send patches or ideas
35; to x264-devel@videolan.org .
36 29
37%ifndef private_prefix30%ifndef private_prefix
38 %define private_prefix x26431 %error private_prefix not defined
39%endif32%endif
40 33
41%ifndef public_prefix34%ifndef public_prefix
@@ -68,12 +61,19 @@ · 26 unmodified lines
68%endif61%endif
69 62
70%define FORMAT_ELF 063%define FORMAT_ELF 0
64%define FORMAT_MACHO 0
71%ifidn __OUTPUT_FORMAT__,elf65%ifidn __OUTPUT_FORMAT__,elf
72 %define FORMAT_ELF 166 %define FORMAT_ELF 1
73%elifidn __OUTPUT_FORMAT__,elf3267%elifidn __OUTPUT_FORMAT__,elf32
74 %define FORMAT_ELF 168 %define FORMAT_ELF 1
75%elifidn __OUTPUT_FORMAT__,elf6469%elifidn __OUTPUT_FORMAT__,elf64
76 %define FORMAT_ELF 170 %define FORMAT_ELF 1
71%elifidn __OUTPUT_FORMAT__,macho
72 %define FORMAT_MACHO 1
73%elifidn __OUTPUT_FORMAT__,macho32
74 %define FORMAT_MACHO 1
75%elifidn __OUTPUT_FORMAT__,macho64
76 %define FORMAT_MACHO 1
77%endif77%endif
78 78
79%ifdef PREFIX79%ifdef PREFIX
@@ -82,6 +82,11 @@ · 2 unmodified lines
82 %define mangle(x) x82 %define mangle(x) x
83%endif83%endif
84 84
85; Use VEX-encoding even in non-AVX functions
86%ifndef FORCE_VEX_ENCODING
87 %define FORCE_VEX_ENCODING 0
88%endif
89
85; aout does not support align=90; aout does not support align=
86; NOTE: This section is out of sync with x264, in order to91; NOTE: This section is out of sync with x264, in order to
87; keep supporting OS/2.92; keep supporting OS/2.
@@ -99,28 +104,27 @@ · 11 unmodified lines
99 %endif104 %endif
100%endmacro105%endmacro
101 106
102%if WIN64107%if ARCH_X86_64
103 %define PIC108 %define PIC 1 ; always use PIC on x86-64
104%elif ARCH_X86_64 == 0
105; x86_32 doesn't require PIC.
106; Some distros prefer shared objects to be PIC, but nothing breaks if
107; the code contains a few textrels, so we'll skip that complexity.
108 %undef PIC
109%endif
110%ifdef PIC
111 default rel109 default rel
110%elifidn __OUTPUT_FORMAT__,win32
111 %define PIC 0 ; PIC isn't used on 32-bit Windows
112%elifndef PIC
113 %define PIC 0
112%endif114%endif
113 115
114%macro CPUNOP 1116%define HAVE_PRIVATE_EXTERN 1
115 %if HAVE_CPUNOP117%ifdef __NASM_VERSION_ID__
116 CPU %1118 %use smartalign
119 %if __NASM_VERSION_ID__ < 0x020e0000 ; 2.14
120 %define HAVE_PRIVATE_EXTERN 0
117 %endif121 %endif
118%endmacro122%endif
119 123
120; Macros to eliminate most code duplication between x86_32 and x86_64:124; Macros to eliminate most code duplication between x86_32 and x86_64:
121; Currently this works only for leaf functions which load all their arguments125; Currently this works only for leaf functions which load all their arguments
122; into registers at the start, and make no other use of the stack. Luckily that126; into registers at the start, and make no other use of the stack. Luckily that
123; covers most of x264's asm.127; covers most use cases.
124 128
125; PROLOGUE:129; PROLOGUE:
126; %1 = number of arguments. loads them from stack if needed.130; %1 = number of arguments. loads them from stack if needed.
@@ -232,6 +236,18 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 105 unmodified lines
232 %define gprsize 4236 %define gprsize 4
233%endif237%endif
234 238
239%macro LEA 2
240%if ARCH_X86_64
241 lea %1, [%2]
242%elif PIC
243 call $+5 ; special-cased to not affect the RSB on most CPU:s
244 pop %1
245 add %1, (%2)-$+1
246%else
247 mov %1, %2
248%endif
249%endmacro
250
235; Repeats an instruction/operation for multiple arguments.251; Repeats an instruction/operation for multiple arguments.
236; Example usage: "REPX {psrlw x, 8}, m0, m1, m2, m3"252; Example usage: "REPX {psrlw x, 8}, m0, m1, m2, m3"
237%macro REPX 2-* ; operation, args253%macro REPX 2-* ; operation, args
@@ -303,6 +319,10 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 65 unmodified lines
303 %endif319 %endif
304%endmacro320%endmacro
305 321
322%if ARCH_X86_64 == 0
323 %define movsxd movifnidn
324%endif
325
306%macro movsxdifnidn 2326%macro movsxdifnidn 2
307 %ifnidn %1, %2327 %ifnidn %1, %2
308 movsxd %1, %2328 movsxd %1, %2
@@ -354,7 +374,46 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 45 unmodified lines
354%define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512)))374%define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512)))
355%define high_mm_regs (16*cpuflag(avx512))375%define high_mm_regs (16*cpuflag(avx512))
356 376
357%macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only)377; Large stack allocations on Windows need to use stack probing in order
378; to guarantee that all stack memory is committed before accessing it.
379; This is done by ensuring that the guard page(s) at the end of the
380; currently committed pages are touched prior to any pages beyond that.
381%if WIN64
382 %assign STACK_PROBE_SIZE 8192
383%elifidn __OUTPUT_FORMAT__, win32
384 %assign STACK_PROBE_SIZE 4096
385%else
386 %assign STACK_PROBE_SIZE 0
387%endif
388
389%macro PROBE_STACK 1 ; stack_size
390 %if STACK_PROBE_SIZE
391 %assign %%i STACK_PROBE_SIZE
392 %rep %1 / STACK_PROBE_SIZE
393 mov eax, [rsp-%%i]
394 %assign %%i %%i+STACK_PROBE_SIZE
395 %endrep
396 %endif
397%endmacro
398
399%macro RESET_STACK_STATE 0
400 %ifidn rstk, rsp
401 %assign stack_offset stack_offset - stack_size_padded
402 %else
403 %xdefine rstk rsp
404 %endif
405 %assign stack_size 0
406 %assign stack_size_padded 0
407 %assign xmm_regs_used 0
408%endmacro
409
410%macro ALLOC_STACK 0-2 0, 0 ; stack_size, n_xmm_regs
411 RESET_STACK_STATE
412 %ifnum %2
413 %if mmsize != 8
414 %assign xmm_regs_used %2
415 %endif
416 %endif
358 %ifnum %1417 %ifnum %1
359 %if %1 != 0418 %if %1 != 0
360 %assign %%pad 0419 %assign %%pad 0
@@ -364,16 +423,14 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 3 unmodified lines
364 %endif423 %endif
365 %if WIN64424 %if WIN64
366 %assign %%pad %%pad + 32 ; shadow space425 %assign %%pad %%pad + 32 ; shadow space
367 %if mmsize != 8426 %if xmm_regs_used > 8
368 %assign xmm_regs_used %2427 %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers
369 %if xmm_regs_used > 8
370 %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers
371 %endif
372 %endif428 %endif
373 %endif429 %endif
374 %if required_stack_alignment <= STACK_ALIGNMENT430 %if required_stack_alignment <= STACK_ALIGNMENT
375 ; maintain the current stack alignment431 ; maintain the current stack alignment
376 %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))432 %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
433 PROBE_STACK stack_size_padded
377 SUB rsp, stack_size_padded434 SUB rsp, stack_size_padded
378 %else435 %else
379 %assign %%reg_num (regs_used - 1)436 %assign %%reg_num (regs_used - 1)
@@ -389,6 +446,7 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 9 unmodified lines
389 %xdefine rstkm rstk446 %xdefine rstkm rstk
390 %endif447 %endif
391 %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1))448 %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1))
449 PROBE_STACK stack_size_padded
392 mov rstk, rsp450 mov rstk, rsp
393 and rsp, ~(required_stack_alignment-1)451 and rsp, ~(required_stack_alignment-1)
394 sub rsp, stack_size_padded452 sub rsp, stack_size_padded
@@ -399,7 +457,7 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 4 unmodified lines
399 %endif457 %endif
400%endmacro458%endmacro
401 459
402%macro SETUP_STACK_POINTER 1460%macro SETUP_STACK_POINTER 0-1 0
403 %ifnum %1461 %ifnum %1
404 %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT462 %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT
405 %if %1 > 0463 %if %1 > 0
@@ -462,35 +520,62 @@ DECLARE_REG 14, R13, 120 · 56 unmodified lines
462 %endif520 %endif
463%endmacro521%endmacro
464 522
465%macro WIN64_PUSH_XMM 0523; Push XMM registers to the stack. If no argument is specified all used register
466 ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated.524; will be pushed, otherwise only push previously unpushed registers.
467 %if xmm_regs_used > 6 + high_mm_regs525%macro WIN64_PUSH_XMM 0-2 ; new_xmm_regs_used, xmm_regs_pushed
468 movaps [rstk + stack_offset + 8], xmm6526 %if mmsize != 8
469 %endif527 %if %0 == 2
470 %if xmm_regs_used > 7 + high_mm_regs528 %assign %%pushed %2
471 movaps [rstk + stack_offset + 24], xmm7529 %assign xmm_regs_used %1
472 %endif530 %elif %0 == 1
473 %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8531 %assign %%pushed xmm_regs_used
474 %if %%xmm_regs_on_stack > 0532 %assign xmm_regs_used %1
475 %assign %%i 8533 %else
476 %rep %%xmm_regs_on_stack534 %assign %%pushed 0
477 movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i535 %endif
478 %assign %%i %%i+1536 ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated.
479 %endrep537 %if %%pushed <= 6 + high_mm_regs && xmm_regs_used > 6 + high_mm_regs
538 movaps [rstk + stack_offset + 8], xmm6
539 %endif
540 %if %%pushed <= 7 + high_mm_regs && xmm_regs_used > 7 + high_mm_regs
541 movaps [rstk + stack_offset + 24], xmm7
542 %endif
543 %assign %%pushed %%pushed - high_mm_regs - 8
544 %if %%pushed < 0
545 %assign %%pushed 0
546 %endif
547 %assign %%regs_to_push xmm_regs_used - %%pushed - high_mm_regs - 8
548 %if %%regs_to_push > 0
549 ASSERT (%%regs_to_push + %%pushed) * 16 <= stack_size_padded - stack_size - 32
550 %assign %%i %%pushed + 8
551 %rep %%regs_to_push
552 movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i
553 %assign %%i %%i+1
554 %endrep
555 %endif
480 %endif556 %endif
481%endmacro557%endmacro
482 558
483%macro WIN64_SPILL_XMM 1559; Allocated stack space for XMM registers and push all, or a subset, of those
484 %assign xmm_regs_used %1560%macro WIN64_SPILL_XMM 1-2 ; xmm_regs_used, xmm_regs_reserved
485 ASSERT xmm_regs_used <= 16 + high_mm_regs561 RESET_STACK_STATE
486 %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8562 %if mmsize != 8
487 %if %%xmm_regs_on_stack > 0563 %assign xmm_regs_used %1
488 ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack.564 ASSERT xmm_regs_used <= 16 + high_mm_regs
489 %assign %%pad %%xmm_regs_on_stack*16 + 32565 %if %0 == 2
490 %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))566 ASSERT %2 >= %1
491 SUB rsp, stack_size_padded567 %assign %%xmm_regs_on_stack %2 - high_mm_regs - 8
568 %else
569 %assign %%xmm_regs_on_stack %1 - high_mm_regs - 8
570 %endif
571 %if %%xmm_regs_on_stack > 0
572 ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack.
573 %assign %%pad %%xmm_regs_on_stack*16 + 32
574 %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
575 SUB rsp, stack_size_padded
576 %endif
577 WIN64_PUSH_XMM
492 %endif578 %endif
493 WIN64_PUSH_XMM
494%endmacro579%endmacro
495 580
496%macro WIN64_RESTORE_XMM_INTERNAL 0581%macro WIN64_RESTORE_XMM_INTERNAL 0
@@ -521,9 +606,7 @@ DECLARE_REG 14, R13, 120 · 24 unmodified lines
521 606
522%macro WIN64_RESTORE_XMM 0607%macro WIN64_RESTORE_XMM 0
523 WIN64_RESTORE_XMM_INTERNAL608 WIN64_RESTORE_XMM_INTERNAL
524 %assign stack_offset (stack_offset-stack_size_padded)609 RESET_STACK_STATE
525 %assign stack_size_padded 0
526 %assign xmm_regs_used 0
527%endmacro610%endmacro
528 611
529%define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs612%define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs
@@ -558,12 +641,11 @@ DECLARE_REG 14, R13, 72 · 28 unmodified lines
558%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...641%macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...
559 %assign num_args %1642 %assign num_args %1
560 %assign regs_used %2643 %assign regs_used %2
561 %assign xmm_regs_used %3
562 ASSERT regs_used >= num_args644 ASSERT regs_used >= num_args
563 SETUP_STACK_POINTER %4645 SETUP_STACK_POINTER %4
564 ASSERT regs_used <= 15646 ASSERT regs_used <= 15
565 PUSH_IF_USED 9, 10, 11, 12, 13, 14647 PUSH_IF_USED 9, 10, 11, 12, 13, 14
566 ALLOC_STACK %4648 ALLOC_STACK %4, %3
567 LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14649 LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14
568 %if %0 > 4650 %if %0 > 4
569 %ifnum %4651 %ifnum %4
@@ -627,7 +709,7 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 57 unmodified lines
627 SETUP_STACK_POINTER %4709 SETUP_STACK_POINTER %4
628 ASSERT regs_used <= 7710 ASSERT regs_used <= 7
629 PUSH_IF_USED 3, 4, 5, 6711 PUSH_IF_USED 3, 4, 5, 6
630 ALLOC_STACK %4712 ALLOC_STACK %4, %3
631 LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6713 LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6
632 %if %0 > 4714 %if %0 > 4
633 %ifnum %4715 %ifnum %4
@@ -660,11 +742,19 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 26 unmodified lines
660%endif ;======================================================================742%endif ;======================================================================
661 743
662%if WIN64 == 0744%if WIN64 == 0
663 %macro WIN64_SPILL_XMM 1745 %macro WIN64_SPILL_XMM 1-2
746 RESET_STACK_STATE
747 %if mmsize != 8
748 %assign xmm_regs_used %1
749 %endif
664 %endmacro750 %endmacro
665 %macro WIN64_RESTORE_XMM 0751 %macro WIN64_RESTORE_XMM 0
752 RESET_STACK_STATE
666 %endmacro753 %endmacro
667 %macro WIN64_PUSH_XMM 0754 %macro WIN64_PUSH_XMM 0-2
755 %if mmsize != 8 && %0 >= 1
756 %assign xmm_regs_used %1
757 %endif
668 %endmacro758 %endmacro
669%endif759%endif
670 760
@@ -705,7 +795,7 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 34 unmodified lines
705 795
706BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp796BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp
707 797
708%macro TAIL_CALL 2 ; callee, is_nonadjacent798%macro TAIL_CALL 1-2 1 ; callee, is_nonadjacent
709 %if has_epilogue799 %if has_epilogue
710 call %1800 call %1
711 RET801 RET
@@ -735,22 +825,25 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 23 unmodified lines
735%endmacro825%endmacro
736%macro cglobal_internal 2-3+826%macro cglobal_internal 2-3+
737 annotate_function_size827 annotate_function_size
738 %if %1
739 %xdefine %%FUNCTION_PREFIX private_prefix
740 %xdefine %%VISIBILITY hidden
741 %else
742 %xdefine %%FUNCTION_PREFIX public_prefix
743 %xdefine %%VISIBILITY
744 %endif
745 %ifndef cglobaled_%2828 %ifndef cglobaled_%2
746 %xdefine %2 mangle(%%FUNCTION_PREFIX %+ _ %+ %2)829 %if %1
830 %xdefine %2 mangle(private_prefix %+ _ %+ %2)
831 %else
832 %xdefine %2 mangle(public_prefix %+ _ %+ %2)
833 %endif
747 %xdefine %2.skip_prologue %2 %+ .skip_prologue834 %xdefine %2.skip_prologue %2 %+ .skip_prologue
748 CAT_XDEFINE cglobaled_, %2, 1835 CAT_XDEFINE cglobaled_, %2, 1
749 %endif836 %endif
750 %xdefine current_function %2837 %xdefine current_function %2
751 %xdefine current_function_section __SECT__838 %xdefine current_function_section __SECT__
752 %if FORMAT_ELF839 %if FORMAT_ELF
753 global %2:function %%VISIBILITY840 %if %1
841 global %2:function hidden
842 %else
843 global %2:function
844 %endif
845 %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN && %1
846 global %2:private_extern
754 %else847 %else
755 global %2848 global %2
756 %endif849 %endif
@@ -771,6 +864,8 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 14 unmodified lines
771%macro cglobal_label 1864%macro cglobal_label 1
772 %if FORMAT_ELF865 %if FORMAT_ELF
773 global current_function %+ %1:function hidden866 global current_function %+ %1:function hidden
867 %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN
868 global current_function %+ %1:private_extern
774 %else869 %else
775 global current_function %+ %1870 global current_function %+ %1
776 %endif871 %endif
@@ -796,15 +891,34 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 19 unmodified lines
796 %xdefine %1 mangle(private_prefix %+ _ %+ %1)891 %xdefine %1 mangle(private_prefix %+ _ %+ %1)
797 %if FORMAT_ELF892 %if FORMAT_ELF
798 global %1:data hidden893 global %1:data hidden
894 %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN
895 global %1:private_extern
799 %else896 %else
800 global %1897 global %1
801 %endif898 %endif
802 %1: %2899 %1: %2
803%endmacro900%endmacro
804 901
805; This is needed for ELF, otherwise the GNU linker assumes the stack is executable by default.
806%if FORMAT_ELF902%if FORMAT_ELF
903 ; The GNU linker assumes the stack is executable by default.
807 [SECTION .note.GNU-stack noalloc noexec nowrite progbits]904 [SECTION .note.GNU-stack noalloc noexec nowrite progbits]
905
906 %ifdef __NASM_VERSION_ID__
907 %if __NASM_VERSION_ID__ >= 0x020e0300 ; 2.14.03
908 %if ARCH_X86_64
909 ; Control-flow Enforcement Technology (CET) properties.
910 [SECTION .note.gnu.property alloc noexec nowrite note align=gprsize]
911 dd 0x00000004 ; n_namesz
912 dd gprsize + 8 ; n_descsz
913 dd 0x00000005 ; n_type = NT_GNU_PROPERTY_TYPE_0
914 db "GNU",0 ; n_name
915 dd 0xc0000002 ; pr_type = GNU_PROPERTY_X86_FEATURE_1_AND
916 dd 0x00000004 ; pr_datasz
917 dd 0x00000002 ; pr_data = GNU_PROPERTY_X86_FEATURE_1_SHSTK
918 dd 0x00000000 ; pr_padding
919 %endif
920 %endif
921 %endif
808%endif922%endif
809 923
810; Tell debuggers how large the function was.924; Tell debuggers how large the function was.
@@ -828,32 +942,34 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 17 unmodified lines
828; cpuflags942; cpuflags
829 943
830%assign cpuflags_mmx (1<<0)944%assign cpuflags_mmx (1<<0)
831%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx945%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx
832%assign cpuflags_3dnow (1<<2) | cpuflags_mmx946%assign cpuflags_3dnow (1<<2) | cpuflags_mmx
833%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow947%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow
834%assign cpuflags_sse (1<<4) | cpuflags_mmx2948%assign cpuflags_sse (1<<4) | cpuflags_mmx2
835%assign cpuflags_sse2 (1<<5) | cpuflags_sse949%assign cpuflags_sse2 (1<<5) | cpuflags_sse
836%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2950%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2
837%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2951%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2
838%assign cpuflags_sse3 (1<<8) | cpuflags_sse2952%assign cpuflags_sse3 (1<<8) | cpuflags_sse2
839%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3953%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3
840%assign cpuflags_sse4 (1<<10)| cpuflags_ssse3954%assign cpuflags_sse4 (1<<10) | cpuflags_ssse3
841%assign cpuflags_sse42 (1<<11)| cpuflags_sse4955%assign cpuflags_sse42 (1<<11) | cpuflags_sse4
842%assign cpuflags_aesni (1<<12)| cpuflags_sse42956%assign cpuflags_aesni (1<<12) | cpuflags_sse42
843%assign cpuflags_avx (1<<13)| cpuflags_sse42957%assign cpuflags_clmul (1<<13) | cpuflags_sse42
844%assign cpuflags_xop (1<<14)| cpuflags_avx958%assign cpuflags_gfni (1<<14) | cpuflags_aesni|cpuflags_clmul
845%assign cpuflags_fma4 (1<<15)| cpuflags_avx959%assign cpuflags_avx (1<<15) | cpuflags_sse42
846%assign cpuflags_fma3 (1<<16)| cpuflags_avx960%assign cpuflags_xop (1<<16) | cpuflags_avx
847%assign cpuflags_bmi1 (1<<17)| cpuflags_avx|cpuflags_lzcnt961%assign cpuflags_fma4 (1<<17) | cpuflags_avx
848%assign cpuflags_bmi2 (1<<18)| cpuflags_bmi1962%assign cpuflags_fma3 (1<<18) | cpuflags_avx
849%assign cpuflags_avx2 (1<<19)| cpuflags_fma3|cpuflags_bmi2963%assign cpuflags_bmi1 (1<<19) | cpuflags_avx|cpuflags_lzcnt
850%assign cpuflags_avx512 (1<<20)| cpuflags_avx2 ; F, CD, BW, DQ, VL964%assign cpuflags_bmi2 (1<<20) | cpuflags_bmi1
851%assign cpuflags_avx512icl (1<<25)| cpuflags_avx512965%assign cpuflags_avx2 (1<<21) | cpuflags_fma3|cpuflags_bmi2
852 966%assign cpuflags_avx512 (1<<22) | cpuflags_avx2 ; F, CD, BW, DQ, VL
853%assign cpuflags_cache32 (1<<21)967%assign cpuflags_avx512icl (1<<23) | cpuflags_avx512|cpuflags_gfni ; VNNI, IFMA, VBMI, VBMI2, VPOPCNTDQ, BITALG, VAES, VPCLMULQDQ
854%assign cpuflags_cache64 (1<<22)968
855%assign cpuflags_aligned (1<<23) ; not a cpu feature, but a function variant969%assign cpuflags_cache32 (1<<24)
856%assign cpuflags_atom (1<<24)970%assign cpuflags_cache64 (1<<25)
971%assign cpuflags_aligned (1<<26) ; not a cpu feature, but a function variant
972%assign cpuflags_atom (1<<27)
857 973
858; Returns a boolean value expressing whether or not the specified cpuflag is enabled.974; Returns a boolean value expressing whether or not the specified cpuflag is enabled.
859%define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1)975%define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1)
@@ -895,9 +1011,17 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 35 unmodified lines
895 %endif1011 %endif
896 1012
897 %if ARCH_X86_64 || cpuflag(sse2)1013 %if ARCH_X86_64 || cpuflag(sse2)
898 CPUNOP amdnop1014 %ifdef __NASM_VERSION_ID__
1015 ALIGNMODE p6
1016 %else
1017 CPU amdnop
1018 %endif
899 %else1019 %else
900 CPUNOP basicnop1020 %ifdef __NASM_VERSION_ID__
1021 ALIGNMODE nop
1022 %else
1023 CPU basicnop
1024 %endif
901 %endif1025 %endif
902%endmacro1026%endmacro
903 1027
@@ -971,7 +1095,7 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 67 unmodified lines
971%endmacro1095%endmacro
972 1096
973%macro INIT_XMM 0-1+1097%macro INIT_XMM 0-1+
974 %assign avx_enabled 01098 %assign avx_enabled FORCE_VEX_ENCODING
975 %define RESET_MM_PERMUTATION INIT_XMM %11099 %define RESET_MM_PERMUTATION INIT_XMM %1
976 %define mmsize 161100 %define mmsize 16
977 %define mova movdqa1101 %define mova movdqa
@@ -983,6 +1107,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 5 unmodified lines
983 %if WIN641107 %if WIN64
984 AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers1108 AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers
985 %endif1109 %endif
1110 %xdefine bcstw 1to8
1111 %xdefine bcstd 1to4
1112 %xdefine bcstq 1to2
986%endmacro1113%endmacro
987 1114
988%macro INIT_YMM 0-1+1115%macro INIT_YMM 0-1+
@@ -996,6 +1123,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 7 unmodified lines
996 INIT_CPUFLAGS %11123 INIT_CPUFLAGS %1
997 DEFINE_MMREGS ymm1124 DEFINE_MMREGS ymm
998 AVX512_MM_PERMUTATION1125 AVX512_MM_PERMUTATION
1126 %xdefine bcstw 1to16
1127 %xdefine bcstd 1to8
1128 %xdefine bcstq 1to4
999%endmacro1129%endmacro
1000 1130
1001%macro INIT_ZMM 0-1+1131%macro INIT_ZMM 0-1+
@@ -1009,6 +1139,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 7 unmodified lines
1009 INIT_CPUFLAGS %11139 INIT_CPUFLAGS %1
1010 DEFINE_MMREGS zmm1140 DEFINE_MMREGS zmm
1011 AVX512_MM_PERMUTATION1141 AVX512_MM_PERMUTATION
1142 %xdefine bcstw 1to32
1143 %xdefine bcstd 1to16
1144 %xdefine bcstq 1to8
1012%endmacro1145%endmacro
1013 1146
1014INIT_XMM1147INIT_XMM
@@ -1106,19 +1239,32 @@ INIT_XMM · 91 unmodified lines
1106 %endif1239 %endif
1107 %assign %%i 01240 %assign %%i 0
1108 %rep num_mmregs1241 %rep num_mmregs
1109 CAT_XDEFINE %%f, %%i, m %+ %%i1242 %xdefine %%tmp m %+ %%i
1243 CAT_XDEFINE %%f, %%i, regnumof %+ %%tmp
1110 %assign %%i %%i+11244 %assign %%i %%i+1
1111 %endrep1245 %endrep
1112%endmacro1246%endmacro
1113 1247
1114%macro LOAD_MM_PERMUTATION 1 ; name to load from1248%macro LOAD_MM_PERMUTATION 0-1 ; name to load from
1115 %ifdef %1_m01249 %if %0
1250 %xdefine %%f %1_m
1251 %else
1252 %xdefine %%f current_function %+ _m
1253 %endif
1254 %xdefine %%tmp %%f %+ 0
1255 %ifnum %%tmp
1256 DEFINE_MMREGS mmtype
1116 %assign %%i 01257 %assign %%i 0
1117 %rep num_mmregs1258 %rep num_mmregs
1118 CAT_XDEFINE m, %%i, %1_m %+ %%i1259 %xdefine %%tmp %%f %+ %%i
1119 CAT_XDEFINE nn, m %+ %%i, %%i1260 CAT_XDEFINE %%m, %%i, m %+ %%tmp
1120 %assign %%i %%i+11261 %assign %%i %%i+1
1121 %endrep1262 %endrep
1263 %rep num_mmregs
1264 %assign %%i %%i-1
1265 CAT_XDEFINE m, %%i, %%m %+ %%i
1266 CAT_XDEFINE nn, m %+ %%i, %%i
1267 %endrep
1122 %endif1268 %endif
1123%endmacro1269%endmacro
1124 1270
@@ -1224,8 +1370,22 @@ INIT_XMM · 99 unmodified lines
1224 %ifdef cpuname1370 %ifdef cpuname
1225 %if notcpuflag(%2)1371 %if notcpuflag(%2)
1226 %error use of ``%1'' %2 instruction in cpuname function: current_function1372 %error use of ``%1'' %2 instruction in cpuname function: current_function
1227 %elif cpuflags_%2 < cpuflags_sse && notcpuflag(sse2) && __sizeofreg > 81373 %elif %3 == 0 && __sizeofreg == 16 && notcpuflag(sse2)
1228 %error use of ``%1'' sse2 instruction in cpuname function: current_function1374 %error use of ``%1'' sse2 instruction in cpuname function: current_function
1375 %elif %3 == 0 && __sizeofreg == 32 && notcpuflag(avx2)
1376 %error use of ``%1'' avx2 instruction in cpuname function: current_function
1377 %elif __sizeofreg == 16 && notcpuflag(sse)
1378 %error use of ``%1'' sse instruction in cpuname function: current_function
1379 %elif __sizeofreg == 32 && notcpuflag(avx)
1380 %error use of ``%1'' avx instruction in cpuname function: current_function
1381 %elif __sizeofreg == 64 && notcpuflag(avx512)
1382 %error use of ``%1'' avx512 instruction in cpuname function: current_function
1383 %elifidn %1, pextrw ; special case because the base instruction is mmx2,
1384 %ifnid %6 ; but sse4 is required for memory operands
1385 %if notcpuflag(sse4)
1386 %error use of ``%1'' sse4 instruction in cpuname function: current_function
1387 %endif
1388 %endif
1229 %endif1389 %endif
1230 %endif1390 %endif
1231 %endif1391 %endif
@@ -1267,11 +1427,79 @@ INIT_XMM · 35 unmodified lines
1267 %1 %6, __src21427 %1 %6, __src2
1268 %endif1428 %endif
1269 %elif %0 >= 91429 %elif %0 >= 9
1270 __instr %6, %7, %8, %91430 %if avx_enabled && __sizeofreg >= 16 && %4 == 1
1431 %ifnnum regnumof%7
1432 %if %3
1433 vmovaps %6, %7
1434 %else
1435 vmovdqa %6, %7
1436 %endif
1437 __instr %6, %6, %8, %9
1438 %else
1439 __instr %6, %7, %8, %9
1440 %endif
1441 %else
1442 __instr %6, %7, %8, %9
1443 %endif
1271 %elif %0 == 81444 %elif %0 == 8
1272 __instr %6, %7, %81445 %if avx_enabled && __sizeofreg >= 16 && %4 == 0
1446 %xdefine __src1 %7
1447 %xdefine __src2 %8
1448 %if %5
1449 %ifnum regnumof%7
1450 %ifnum regnumof%8
1451 %if regnumof%7 < 8 && regnumof%8 >= 8 && regnumof%8 < 16 && sizeof%8 <= 32
1452 ; Most VEX-encoded instructions require an additional byte to encode when
1453 ; src2 is a high register (e.g. m8..15). If the instruction is commutative
1454 ; we can swap src1 and src2 when doing so reduces the instruction length.
1455 %xdefine __src1 %8
1456 %xdefine __src2 %7
1457 %endif
1458 %endif
1459 %elifnum regnumof%8 ; put memory operands in src2 when possible
1460 %xdefine __src1 %8
1461 %xdefine __src2 %7
1462 %else
1463 %assign __emulate_avx 1
1464 %endif
1465 %elifnnum regnumof%7
1466 ; EVEX allows imm8 shift instructions to be used with memory operands,
1467 ; but VEX does not. This handles those special cases.
1468 %ifnnum %8
1469 %assign __emulate_avx 1
1470 %elif notcpuflag(avx512)
1471 %assign __emulate_avx 1
1472 %endif
1473 %endif
1474 %if __emulate_avx ; a separate load is required
1475 %if %3
1476 vmovaps %6, %7
1477 %else
1478 vmovdqa %6, %7
1479 %endif
1480 __instr %6, %6, %8
1481 %else
1482 __instr %6, __src1, __src2
1483 %endif
1484 %else
1485 __instr %6, %7, %8
1486 %endif
1273 %elif %0 == 71487 %elif %0 == 7
1274 __instr %6, %71488 %if avx_enabled && __sizeofreg >= 16 && %5
1489 %xdefine __src1 %6
1490 %xdefine __src2 %7
1491 %ifnum regnumof%6
1492 %ifnum regnumof%7
1493 %if regnumof%6 < 8 && regnumof%7 >= 8 && regnumof%7 < 16 && sizeof%7 <= 32
1494 %xdefine __src1 %7
1495 %xdefine __src2 %6
1496 %endif
1497 %endif
1498 %endif
1499 __instr %6, __src1, __src2
1500 %else
1501 __instr %6, %7
1502 %endif
1275 %else1503 %else
1276 __instr %61504 __instr %6
1277 %endif1505 %endif
@@ -1318,8 +1546,8 @@ AVX_INSTR andpd, sse2, 1, 0, 1 · 40 unmodified lines
1318AVX_INSTR andps, sse, 1, 0, 11546AVX_INSTR andps, sse, 1, 0, 1
1319AVX_INSTR blendpd, sse4, 1, 1, 01547AVX_INSTR blendpd, sse4, 1, 1, 0
1320AVX_INSTR blendps, sse4, 1, 1, 01548AVX_INSTR blendps, sse4, 1, 1, 0
1321AVX_INSTR blendvpd, sse4 ; can't be emulated1549AVX_INSTR blendvpd, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding
1322AVX_INSTR blendvps, sse4 ; can't be emulated1550AVX_INSTR blendvps, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding
1323AVX_INSTR cmpeqpd, sse2, 1, 0, 11551AVX_INSTR cmpeqpd, sse2, 1, 0, 1
1324AVX_INSTR cmpeqps, sse, 1, 0, 11552AVX_INSTR cmpeqps, sse, 1, 0, 1
1325AVX_INSTR cmpeqsd, sse2, 1, 0, 01553AVX_INSTR cmpeqsd, sse2, 1, 0, 0
@@ -1356,38 +1584,41 @@ AVX_INSTR cmpunordpd, sse2, 1, 0, 1 · 30 unmodified lines
1356AVX_INSTR cmpunordps, sse, 1, 0, 11584AVX_INSTR cmpunordps, sse, 1, 0, 1
1357AVX_INSTR cmpunordsd, sse2, 1, 0, 01585AVX_INSTR cmpunordsd, sse2, 1, 0, 0
1358AVX_INSTR cmpunordss, sse, 1, 0, 01586AVX_INSTR cmpunordss, sse, 1, 0, 0
1359AVX_INSTR comisd, sse21587AVX_INSTR comisd, sse2, 1
1360AVX_INSTR comiss, sse1588AVX_INSTR comiss, sse, 1
1361AVX_INSTR cvtdq2pd, sse21589AVX_INSTR cvtdq2pd, sse2, 1
1362AVX_INSTR cvtdq2ps, sse21590AVX_INSTR cvtdq2ps, sse2, 1
1363AVX_INSTR cvtpd2dq, sse21591AVX_INSTR cvtpd2dq, sse2, 1
1364AVX_INSTR cvtpd2ps, sse21592AVX_INSTR cvtpd2ps, sse2, 1
1365AVX_INSTR cvtps2dq, sse21593AVX_INSTR cvtps2dq, sse2, 1
1366AVX_INSTR cvtps2pd, sse21594AVX_INSTR cvtps2pd, sse2, 1
1367AVX_INSTR cvtsd2si, sse21595AVX_INSTR cvtsd2si, sse2, 1
1368AVX_INSTR cvtsd2ss, sse2, 1, 0, 01596AVX_INSTR cvtsd2ss, sse2, 1, 0, 0
1369AVX_INSTR cvtsi2sd, sse2, 1, 0, 01597AVX_INSTR cvtsi2sd, sse2, 1, 0, 0
1370AVX_INSTR cvtsi2ss, sse, 1, 0, 01598AVX_INSTR cvtsi2ss, sse, 1, 0, 0
1371AVX_INSTR cvtss2sd, sse2, 1, 0, 01599AVX_INSTR cvtss2sd, sse2, 1, 0, 0
1372AVX_INSTR cvtss2si, sse1600AVX_INSTR cvtss2si, sse, 1
1373AVX_INSTR cvttpd2dq, sse21601AVX_INSTR cvttpd2dq, sse2, 1
1374AVX_INSTR cvttps2dq, sse21602AVX_INSTR cvttps2dq, sse2, 1
1375AVX_INSTR cvttsd2si, sse21603AVX_INSTR cvttsd2si, sse2, 1
1376AVX_INSTR cvttss2si, sse1604AVX_INSTR cvttss2si, sse, 1
1377AVX_INSTR divpd, sse2, 1, 0, 01605AVX_INSTR divpd, sse2, 1, 0, 0
1378AVX_INSTR divps, sse, 1, 0, 01606AVX_INSTR divps, sse, 1, 0, 0
1379AVX_INSTR divsd, sse2, 1, 0, 01607AVX_INSTR divsd, sse2, 1, 0, 0
1380AVX_INSTR divss, sse, 1, 0, 01608AVX_INSTR divss, sse, 1, 0, 0
1381AVX_INSTR dppd, sse4, 1, 1, 01609AVX_INSTR dppd, sse4, 1, 1, 0
1382AVX_INSTR dpps, sse4, 1, 1, 01610AVX_INSTR dpps, sse4, 1, 1, 0
1383AVX_INSTR extractps, sse41611AVX_INSTR extractps, sse4, 1
1612AVX_INSTR gf2p8affineinvqb, gfni, 0, 1, 0
1613AVX_INSTR gf2p8affineqb, gfni, 0, 1, 0
1614AVX_INSTR gf2p8mulb, gfni, 0, 0, 0
1384AVX_INSTR haddpd, sse3, 1, 0, 01615AVX_INSTR haddpd, sse3, 1, 0, 0
1385AVX_INSTR haddps, sse3, 1, 0, 01616AVX_INSTR haddps, sse3, 1, 0, 0
1386AVX_INSTR hsubpd, sse3, 1, 0, 01617AVX_INSTR hsubpd, sse3, 1, 0, 0
1387AVX_INSTR hsubps, sse3, 1, 0, 01618AVX_INSTR hsubps, sse3, 1, 0, 0
1388AVX_INSTR insertps, sse4, 1, 1, 01619AVX_INSTR insertps, sse4, 1, 1, 0
1389AVX_INSTR lddqu, sse31620AVX_INSTR lddqu, sse3
1390AVX_INSTR ldmxcsr, sse1621AVX_INSTR ldmxcsr, sse, 1
1391AVX_INSTR maskmovdqu, sse21622AVX_INSTR maskmovdqu, sse2
1392AVX_INSTR maxpd, sse2, 1, 0, 11623AVX_INSTR maxpd, sse2, 1, 0, 1
1393AVX_INSTR maxps, sse, 1, 0, 11624AVX_INSTR maxps, sse, 1, 0, 1
@@ -1397,10 +1628,10 @@ AVX_INSTR minpd, sse2, 1, 0, 1 · 3 unmodified lines
1397AVX_INSTR minps, sse, 1, 0, 11628AVX_INSTR minps, sse, 1, 0, 1
1398AVX_INSTR minsd, sse2, 1, 0, 01629AVX_INSTR minsd, sse2, 1, 0, 0
1399AVX_INSTR minss, sse, 1, 0, 01630AVX_INSTR minss, sse, 1, 0, 0
1400AVX_INSTR movapd, sse21631AVX_INSTR movapd, sse2, 1
1401AVX_INSTR movaps, sse1632AVX_INSTR movaps, sse, 1
1402AVX_INSTR movd, mmx1633AVX_INSTR movd, mmx
1403AVX_INSTR movddup, sse31634AVX_INSTR movddup, sse3, 1
1404AVX_INSTR movdqa, sse21635AVX_INSTR movdqa, sse2
1405AVX_INSTR movdqu, sse21636AVX_INSTR movdqu, sse2
1406AVX_INSTR movhlps, sse, 1, 0, 01637AVX_INSTR movhlps, sse, 1, 0, 0
@@ -1409,19 +1640,19 @@ AVX_INSTR movhps, sse, 1, 0, 0 · 2 unmodified lines
1409AVX_INSTR movlhps, sse, 1, 0, 01640AVX_INSTR movlhps, sse, 1, 0, 0
1410AVX_INSTR movlpd, sse2, 1, 0, 01641AVX_INSTR movlpd, sse2, 1, 0, 0
1411AVX_INSTR movlps, sse, 1, 0, 01642AVX_INSTR movlps, sse, 1, 0, 0
1412AVX_INSTR movmskpd, sse21643AVX_INSTR movmskpd, sse2, 1
1413AVX_INSTR movmskps, sse1644AVX_INSTR movmskps, sse, 1
1414AVX_INSTR movntdq, sse21645AVX_INSTR movntdq, sse2
1415AVX_INSTR movntdqa, sse41646AVX_INSTR movntdqa, sse4
1416AVX_INSTR movntpd, sse21647AVX_INSTR movntpd, sse2, 1
1417AVX_INSTR movntps, sse1648AVX_INSTR movntps, sse, 1
1418AVX_INSTR movq, mmx1649AVX_INSTR movq, mmx
1419AVX_INSTR movsd, sse2, 1, 0, 01650AVX_INSTR movsd, sse2, 1, 0, 0
1420AVX_INSTR movshdup, sse31651AVX_INSTR movshdup, sse3, 1
1421AVX_INSTR movsldup, sse31652AVX_INSTR movsldup, sse3, 1
1422AVX_INSTR movss, sse, 1, 0, 01653AVX_INSTR movss, sse, 1, 0, 0
1423AVX_INSTR movupd, sse21654AVX_INSTR movupd, sse2, 1
1424AVX_INSTR movups, sse1655AVX_INSTR movups, sse, 1
1425AVX_INSTR mpsadbw, sse4, 0, 1, 01656AVX_INSTR mpsadbw, sse4, 0, 1, 0
1426AVX_INSTR mulpd, sse2, 1, 0, 11657AVX_INSTR mulpd, sse2, 1, 0, 1
1427AVX_INSTR mulps, sse, 1, 0, 11658AVX_INSTR mulps, sse, 1, 0, 1
@@ -1432,90 +1663,90 @@ AVX_INSTR orps, sse, 1, 0, 1 · 4 unmodified lines
1432AVX_INSTR pabsb, ssse31663AVX_INSTR pabsb, ssse3
1433AVX_INSTR pabsd, ssse31664AVX_INSTR pabsd, ssse3
1434AVX_INSTR pabsw, ssse31665AVX_INSTR pabsw, ssse3
1435AVX_INSTR packsswb, mmx, 0, 0, 0
1436AVX_INSTR packssdw, mmx, 0, 0, 01666AVX_INSTR packssdw, mmx, 0, 0, 0
1437AVX_INSTR packuswb, mmx, 0, 0, 01667AVX_INSTR packsswb, mmx, 0, 0, 0
1438AVX_INSTR packusdw, sse4, 0, 0, 01668AVX_INSTR packusdw, sse4, 0, 0, 0
1669AVX_INSTR packuswb, mmx, 0, 0, 0
1439AVX_INSTR paddb, mmx, 0, 0, 11670AVX_INSTR paddb, mmx, 0, 0, 1
1440AVX_INSTR paddw, mmx, 0, 0, 1
1441AVX_INSTR paddd, mmx, 0, 0, 11671AVX_INSTR paddd, mmx, 0, 0, 1
1442AVX_INSTR paddq, sse2, 0, 0, 11672AVX_INSTR paddq, sse2, 0, 0, 1
1443AVX_INSTR paddsb, mmx, 0, 0, 11673AVX_INSTR paddsb, mmx, 0, 0, 1
1444AVX_INSTR paddsw, mmx, 0, 0, 11674AVX_INSTR paddsw, mmx, 0, 0, 1
1445AVX_INSTR paddusb, mmx, 0, 0, 11675AVX_INSTR paddusb, mmx, 0, 0, 1
1446AVX_INSTR paddusw, mmx, 0, 0, 11676AVX_INSTR paddusw, mmx, 0, 0, 1
1677AVX_INSTR paddw, mmx, 0, 0, 1
1447AVX_INSTR palignr, ssse3, 0, 1, 01678AVX_INSTR palignr, ssse3, 0, 1, 0
1448AVX_INSTR pand, mmx, 0, 0, 11679AVX_INSTR pand, mmx, 0, 0, 1
1449AVX_INSTR pandn, mmx, 0, 0, 01680AVX_INSTR pandn, mmx, 0, 0, 0
1450AVX_INSTR pavgb, mmx2, 0, 0, 11681AVX_INSTR pavgb, mmx2, 0, 0, 1
1451AVX_INSTR pavgw, mmx2, 0, 0, 11682AVX_INSTR pavgw, mmx2, 0, 0, 1
1452AVX_INSTR pblendvb, sse4 ; can't be emulated1683AVX_INSTR pblendvb, sse4, 0, 1, 0 ; last operand must be xmm0 with legacy encoding
1453AVX_INSTR pblendw, sse4, 0, 1, 01684AVX_INSTR pblendw, sse4, 0, 1, 0
1454AVX_INSTR pclmulqdq, fnord, 0, 1, 01685AVX_INSTR pclmulhqhqdq, clmul, 0, 0, 0
1455AVX_INSTR pclmulhqhqdq, fnord, 0, 0, 01686AVX_INSTR pclmulhqlqdq, clmul, 0, 0, 0
1456AVX_INSTR pclmulhqlqdq, fnord, 0, 0, 01687AVX_INSTR pclmullqhqdq, clmul, 0, 0, 0
1457AVX_INSTR pclmullqhqdq, fnord, 0, 0, 01688AVX_INSTR pclmullqlqdq, clmul, 0, 0, 0
1458AVX_INSTR pclmullqlqdq, fnord, 0, 0, 01689AVX_INSTR pclmulqdq, clmul, 0, 1, 0
1459AVX_INSTR pcmpestri, sse42
1460AVX_INSTR pcmpestrm, sse42
1461AVX_INSTR pcmpistri, sse42
1462AVX_INSTR pcmpistrm, sse42
1463AVX_INSTR pcmpeqb, mmx, 0, 0, 11690AVX_INSTR pcmpeqb, mmx, 0, 0, 1
1464AVX_INSTR pcmpeqw, mmx, 0, 0, 1
1465AVX_INSTR pcmpeqd, mmx, 0, 0, 11691AVX_INSTR pcmpeqd, mmx, 0, 0, 1
1466AVX_INSTR pcmpeqq, sse4, 0, 0, 11692AVX_INSTR pcmpeqq, sse4, 0, 0, 1
1693AVX_INSTR pcmpeqw, mmx, 0, 0, 1
1694AVX_INSTR pcmpestri, sse42
1695AVX_INSTR pcmpestrm, sse42
1467AVX_INSTR pcmpgtb, mmx, 0, 0, 01696AVX_INSTR pcmpgtb, mmx, 0, 0, 0
1468AVX_INSTR pcmpgtw, mmx, 0, 0, 0
1469AVX_INSTR pcmpgtd, mmx, 0, 0, 01697AVX_INSTR pcmpgtd, mmx, 0, 0, 0
1470AVX_INSTR pcmpgtq, sse42, 0, 0, 01698AVX_INSTR pcmpgtq, sse42, 0, 0, 0
1699AVX_INSTR pcmpgtw, mmx, 0, 0, 0
1700AVX_INSTR pcmpistri, sse42
1701AVX_INSTR pcmpistrm, sse42
1471AVX_INSTR pextrb, sse41702AVX_INSTR pextrb, sse4
1472AVX_INSTR pextrd, sse41703AVX_INSTR pextrd, sse4
1473AVX_INSTR pextrq, sse41704AVX_INSTR pextrq, sse4
1474AVX_INSTR pextrw, mmx21705AVX_INSTR pextrw, mmx2
1475AVX_INSTR phaddw, ssse3, 0, 0, 0
1476AVX_INSTR phaddd, ssse3, 0, 0, 01706AVX_INSTR phaddd, ssse3, 0, 0, 0
1477AVX_INSTR phaddsw, ssse3, 0, 0, 01707AVX_INSTR phaddsw, ssse3, 0, 0, 0
1708AVX_INSTR phaddw, ssse3, 0, 0, 0
1478AVX_INSTR phminposuw, sse41709AVX_INSTR phminposuw, sse4
1479AVX_INSTR phsubw, ssse3, 0, 0, 0
1480AVX_INSTR phsubd, ssse3, 0, 0, 01710AVX_INSTR phsubd, ssse3, 0, 0, 0
1481AVX_INSTR phsubsw, ssse3, 0, 0, 01711AVX_INSTR phsubsw, ssse3, 0, 0, 0
1712AVX_INSTR phsubw, ssse3, 0, 0, 0
1482AVX_INSTR pinsrb, sse4, 0, 1, 01713AVX_INSTR pinsrb, sse4, 0, 1, 0
1483AVX_INSTR pinsrd, sse4, 0, 1, 01714AVX_INSTR pinsrd, sse4, 0, 1, 0
1484AVX_INSTR pinsrq, sse4, 0, 1, 01715AVX_INSTR pinsrq, sse4, 0, 1, 0
1485AVX_INSTR pinsrw, mmx2, 0, 1, 01716AVX_INSTR pinsrw, mmx2, 0, 1, 0
1486AVX_INSTR pmaddwd, mmx, 0, 0, 1
1487AVX_INSTR pmaddubsw, ssse3, 0, 0, 01717AVX_INSTR pmaddubsw, ssse3, 0, 0, 0
1718AVX_INSTR pmaddwd, mmx, 0, 0, 1
1488AVX_INSTR pmaxsb, sse4, 0, 0, 11719AVX_INSTR pmaxsb, sse4, 0, 0, 1
1489AVX_INSTR pmaxsw, mmx2, 0, 0, 1
1490AVX_INSTR pmaxsd, sse4, 0, 0, 11720AVX_INSTR pmaxsd, sse4, 0, 0, 1
1721AVX_INSTR pmaxsw, mmx2, 0, 0, 1
1491AVX_INSTR pmaxub, mmx2, 0, 0, 11722AVX_INSTR pmaxub, mmx2, 0, 0, 1
1492AVX_INSTR pmaxuw, sse4, 0, 0, 1
1493AVX_INSTR pmaxud, sse4, 0, 0, 11723AVX_INSTR pmaxud, sse4, 0, 0, 1
1724AVX_INSTR pmaxuw, sse4, 0, 0, 1
1494AVX_INSTR pminsb, sse4, 0, 0, 11725AVX_INSTR pminsb, sse4, 0, 0, 1
1495AVX_INSTR pminsw, mmx2, 0, 0, 1
1496AVX_INSTR pminsd, sse4, 0, 0, 11726AVX_INSTR pminsd, sse4, 0, 0, 1
1727AVX_INSTR pminsw, mmx2, 0, 0, 1
1497AVX_INSTR pminub, mmx2, 0, 0, 11728AVX_INSTR pminub, mmx2, 0, 0, 1
1498AVX_INSTR pminuw, sse4, 0, 0, 1
1499AVX_INSTR pminud, sse4, 0, 0, 11729AVX_INSTR pminud, sse4, 0, 0, 1
1730AVX_INSTR pminuw, sse4, 0, 0, 1
1500AVX_INSTR pmovmskb, mmx21731AVX_INSTR pmovmskb, mmx2
1501AVX_INSTR pmovsxbw, sse4
1502AVX_INSTR pmovsxbd, sse41732AVX_INSTR pmovsxbd, sse4
1503AVX_INSTR pmovsxbq, sse41733AVX_INSTR pmovsxbq, sse4
1734AVX_INSTR pmovsxbw, sse4
1735AVX_INSTR pmovsxdq, sse4
1504AVX_INSTR pmovsxwd, sse41736AVX_INSTR pmovsxwd, sse4
1505AVX_INSTR pmovsxwq, sse41737AVX_INSTR pmovsxwq, sse4
1506AVX_INSTR pmovsxdq, sse4
1507AVX_INSTR pmovzxbw, sse4
1508AVX_INSTR pmovzxbd, sse41738AVX_INSTR pmovzxbd, sse4
1509AVX_INSTR pmovzxbq, sse41739AVX_INSTR pmovzxbq, sse4
1740AVX_INSTR pmovzxbw, sse4
1741AVX_INSTR pmovzxdq, sse4
1510AVX_INSTR pmovzxwd, sse41742AVX_INSTR pmovzxwd, sse4
1511AVX_INSTR pmovzxwq, sse41743AVX_INSTR pmovzxwq, sse4
1512AVX_INSTR pmovzxdq, sse4
1513AVX_INSTR pmuldq, sse4, 0, 0, 11744AVX_INSTR pmuldq, sse4, 0, 0, 1
1514AVX_INSTR pmulhrsw, ssse3, 0, 0, 11745AVX_INSTR pmulhrsw, ssse3, 0, 0, 1
1515AVX_INSTR pmulhuw, mmx2, 0, 0, 11746AVX_INSTR pmulhuw, mmx2, 0, 0, 1
1516AVX_INSTR pmulhw, mmx, 0, 0, 11747AVX_INSTR pmulhw, mmx, 0, 0, 1
1517AVX_INSTR pmullw, mmx, 0, 0, 1
1518AVX_INSTR pmulld, sse4, 0, 0, 11748AVX_INSTR pmulld, sse4, 0, 0, 1
1749AVX_INSTR pmullw, mmx, 0, 0, 1
1519AVX_INSTR pmuludq, sse2, 0, 0, 11750AVX_INSTR pmuludq, sse2, 0, 0, 1
1520AVX_INSTR por, mmx, 0, 0, 11751AVX_INSTR por, mmx, 0, 0, 1
1521AVX_INSTR psadbw, mmx2, 0, 0, 11752AVX_INSTR psadbw, mmx2, 0, 0, 1
@@ -1524,57 +1755,57 @@ AVX_INSTR pshufd, sse2 · 2 unmodified lines
1524AVX_INSTR pshufhw, sse21755AVX_INSTR pshufhw, sse2
1525AVX_INSTR pshuflw, sse21756AVX_INSTR pshuflw, sse2
1526AVX_INSTR psignb, ssse3, 0, 0, 01757AVX_INSTR psignb, ssse3, 0, 0, 0
1527AVX_INSTR psignw, ssse3, 0, 0, 0
1528AVX_INSTR psignd, ssse3, 0, 0, 01758AVX_INSTR psignd, ssse3, 0, 0, 0
1529AVX_INSTR psllw, mmx, 0, 0, 01759AVX_INSTR psignw, ssse3, 0, 0, 0
1530AVX_INSTR pslld, mmx, 0, 0, 01760AVX_INSTR pslld, mmx, 0, 0, 0
1531AVX_INSTR psllq, mmx, 0, 0, 0
1532AVX_INSTR pslldq, sse2, 0, 0, 01761AVX_INSTR pslldq, sse2, 0, 0, 0
1533AVX_INSTR psraw, mmx, 0, 0, 01762AVX_INSTR psllq, mmx, 0, 0, 0
1763AVX_INSTR psllw, mmx, 0, 0, 0
1534AVX_INSTR psrad, mmx, 0, 0, 01764AVX_INSTR psrad, mmx, 0, 0, 0
1535AVX_INSTR psrlw, mmx, 0, 0, 01765AVX_INSTR psraw, mmx, 0, 0, 0
1536AVX_INSTR psrld, mmx, 0, 0, 01766AVX_INSTR psrld, mmx, 0, 0, 0
1537AVX_INSTR psrlq, mmx, 0, 0, 0
1538AVX_INSTR psrldq, sse2, 0, 0, 01767AVX_INSTR psrldq, sse2, 0, 0, 0
1768AVX_INSTR psrlq, mmx, 0, 0, 0
1769AVX_INSTR psrlw, mmx, 0, 0, 0
1539AVX_INSTR psubb, mmx, 0, 0, 01770AVX_INSTR psubb, mmx, 0, 0, 0
1540AVX_INSTR psubw, mmx, 0, 0, 0
1541AVX_INSTR psubd, mmx, 0, 0, 01771AVX_INSTR psubd, mmx, 0, 0, 0
1542AVX_INSTR psubq, sse2, 0, 0, 01772AVX_INSTR psubq, sse2, 0, 0, 0
1543AVX_INSTR psubsb, mmx, 0, 0, 01773AVX_INSTR psubsb, mmx, 0, 0, 0
1544AVX_INSTR psubsw, mmx, 0, 0, 01774AVX_INSTR psubsw, mmx, 0, 0, 0
1545AVX_INSTR psubusb, mmx, 0, 0, 01775AVX_INSTR psubusb, mmx, 0, 0, 0
1546AVX_INSTR psubusw, mmx, 0, 0, 01776AVX_INSTR psubusw, mmx, 0, 0, 0
1777AVX_INSTR psubw, mmx, 0, 0, 0
1547AVX_INSTR ptest, sse41778AVX_INSTR ptest, sse4
1548AVX_INSTR punpckhbw, mmx, 0, 0, 01779AVX_INSTR punpckhbw, mmx, 0, 0, 0
1549AVX_INSTR punpckhwd, mmx, 0, 0, 0
1550AVX_INSTR punpckhdq, mmx, 0, 0, 01780AVX_INSTR punpckhdq, mmx, 0, 0, 0
1551AVX_INSTR punpckhqdq, sse2, 0, 0, 01781AVX_INSTR punpckhqdq, sse2, 0, 0, 0
1782AVX_INSTR punpckhwd, mmx, 0, 0, 0
1552AVX_INSTR punpcklbw, mmx, 0, 0, 01783AVX_INSTR punpcklbw, mmx, 0, 0, 0
1553AVX_INSTR punpcklwd, mmx, 0, 0, 0
1554AVX_INSTR punpckldq, mmx, 0, 0, 01784AVX_INSTR punpckldq, mmx, 0, 0, 0
1555AVX_INSTR punpcklqdq, sse2, 0, 0, 01785AVX_INSTR punpcklqdq, sse2, 0, 0, 0
1786AVX_INSTR punpcklwd, mmx, 0, 0, 0
1556AVX_INSTR pxor, mmx, 0, 0, 11787AVX_INSTR pxor, mmx, 0, 0, 1
1557AVX_INSTR rcpps, sse1788AVX_INSTR rcpps, sse, 1
1558AVX_INSTR rcpss, sse, 1, 0, 01789AVX_INSTR rcpss, sse, 1, 0, 0
1559AVX_INSTR roundpd, sse41790AVX_INSTR roundpd, sse4, 1
1560AVX_INSTR roundps, sse41791AVX_INSTR roundps, sse4, 1
1561AVX_INSTR roundsd, sse4, 1, 1, 01792AVX_INSTR roundsd, sse4, 1, 1, 0
1562AVX_INSTR roundss, sse4, 1, 1, 01793AVX_INSTR roundss, sse4, 1, 1, 0
1563AVX_INSTR rsqrtps, sse1794AVX_INSTR rsqrtps, sse, 1
1564AVX_INSTR rsqrtss, sse, 1, 0, 01795AVX_INSTR rsqrtss, sse, 1, 0, 0
1565AVX_INSTR shufpd, sse2, 1, 1, 01796AVX_INSTR shufpd, sse2, 1, 1, 0
1566AVX_INSTR shufps, sse, 1, 1, 01797AVX_INSTR shufps, sse, 1, 1, 0
1567AVX_INSTR sqrtpd, sse21798AVX_INSTR sqrtpd, sse2, 1
1568AVX_INSTR sqrtps, sse1799AVX_INSTR sqrtps, sse, 1
1569AVX_INSTR sqrtsd, sse2, 1, 0, 01800AVX_INSTR sqrtsd, sse2, 1, 0, 0
1570AVX_INSTR sqrtss, sse, 1, 0, 01801AVX_INSTR sqrtss, sse, 1, 0, 0
1571AVX_INSTR stmxcsr, sse1802AVX_INSTR stmxcsr, sse, 1
1572AVX_INSTR subpd, sse2, 1, 0, 01803AVX_INSTR subpd, sse2, 1, 0, 0
1573AVX_INSTR subps, sse, 1, 0, 01804AVX_INSTR subps, sse, 1, 0, 0
1574AVX_INSTR subsd, sse2, 1, 0, 01805AVX_INSTR subsd, sse2, 1, 0, 0
1575AVX_INSTR subss, sse, 1, 0, 01806AVX_INSTR subss, sse, 1, 0, 0
1576AVX_INSTR ucomisd, sse21807AVX_INSTR ucomisd, sse2, 1
1577AVX_INSTR ucomiss, sse1808AVX_INSTR ucomiss, sse, 1
1578AVX_INSTR unpckhpd, sse2, 1, 0, 01809AVX_INSTR unpckhpd, sse2, 1, 0, 0
1579AVX_INSTR unpckhps, sse, 1, 0, 01810AVX_INSTR unpckhps, sse, 1, 0, 0
1580AVX_INSTR unpcklpd, sse2, 1, 0, 01811AVX_INSTR unpcklpd, sse2, 1, 0, 0
@@ -1584,8 +1815,41 @@ AVX_INSTR xorps, sse, 1, 0, 1 · 3 unmodified lines
1584 1815
1585; 3DNow instructions, for sharing code between AVX, SSE and 3DN1816; 3DNow instructions, for sharing code between AVX, SSE and 3DN
1586AVX_INSTR pfadd, 3dnow, 1, 0, 11817AVX_INSTR pfadd, 3dnow, 1, 0, 1
1587AVX_INSTR pfsub, 3dnow, 1, 0, 0
1588AVX_INSTR pfmul, 3dnow, 1, 0, 11818AVX_INSTR pfmul, 3dnow, 1, 0, 1
1819AVX_INSTR pfsub, 3dnow, 1, 0, 0
1820
1821;%1 == instruction
1822;%2 == minimal instruction set
1823%macro GPR_INSTR 2
1824 %macro %1 2-5 fnord, %1, %2
1825 %ifdef cpuname
1826 %if notcpuflag(%5)
1827 %error use of ``%4'' %5 instruction in cpuname function: current_function
1828 %endif
1829 %endif
1830 %ifidn %3, fnord
1831 %4 %1, %2
1832 %else
1833 %4 %1, %2, %3
1834 %endif
1835 %endmacro
1836%endmacro
1837
1838GPR_INSTR andn, bmi1
1839GPR_INSTR bextr, bmi1
1840GPR_INSTR blsi, bmi1
1841GPR_INSTR blsmsk, bmi1
1842GPR_INSTR blsr, bmi1
1843GPR_INSTR bzhi, bmi2
1844GPR_INSTR crc32, sse42
1845GPR_INSTR mulx, bmi2
1846GPR_INSTR pdep, bmi2
1847GPR_INSTR pext, bmi2
1848GPR_INSTR popcnt, sse42
1849GPR_INSTR rorx, bmi2
1850GPR_INSTR sarx, bmi2
1851GPR_INSTR shlx, bmi2
1852GPR_INSTR shrx, bmi2
1589 1853
1590; base-4 constants for shuffles1854; base-4 constants for shuffles
1591%assign i 01855%assign i 0
@@ -1618,15 +1882,11 @@ AVX_INSTR pfmul, 3dnow, 1, 0, 1 · 26 unmodified lines
1618 %endmacro1882 %endmacro
1619%endmacro1883%endmacro
1620 1884
1621FMA_INSTR pmacsww, pmullw, paddw1885FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation
1622FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation1886FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation
1623FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation1887FMA_INSTR pmacsww, pmullw, paddw
1624FMA_INSTR pmadcswd, pmaddwd, paddd1888FMA_INSTR pmadcswd, pmaddwd, paddd
1625 1889
1626; tzcnt is equivalent to "rep bsf" and is backwards-compatible with bsf.
1627; This lets us use tzcnt without bumping the yasm version requirement yet.
1628%define tzcnt rep bsf
1629
1630; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax.1890; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax.
1631; FMA3 is only possible if dst is the same as one of the src registers.1891; FMA3 is only possible if dst is the same as one of the src registers.
1632; Either src2 or src3 can be a memory operand.1892; Either src2 or src3 can be a memory operand.
@@ -1687,6 +1947,11 @@ FMA4_INSTR fnmsub, pd, ps, sd, ss · 54 unmodified lines
1687 %assign %%evex_required 11947 %assign %%evex_required 1
1688 %endif1948 %endif
1689 %endif1949 %endif
1950 %ifnum regnumof%3
1951 %if regnumof%3 >= 16 || sizeof%3 > 32
1952 %assign %%evex_required 1
1953 %endif
1954 %endif
1690 %if %%evex_required1955 %if %%evex_required
1691 %6 %%args1956 %6 %%args
1692 %else1957 %else
@@ -1711,16 +1976,3 @@ EVEX_INSTR vrcpps, vrcp14ps, 1 ; EVEX versions have higher precision · 18 unmodified lines
1711EVEX_INSTR vrcpss, vrcp14ss, 11976EVEX_INSTR vrcpss, vrcp14ss, 1
1712EVEX_INSTR vrsqrtps, vrsqrt14ps, 11977EVEX_INSTR vrsqrtps, vrsqrt14ps, 1
1713EVEX_INSTR vrsqrtss, vrsqrt14ss, 11978EVEX_INSTR vrsqrtss, vrsqrt14ss, 1
1714
1715; workaround: vpbroadcastq is broken in x86_32 due to a yasm bug (fixed in 1.3.0)
1716%ifdef __YASM_VER__
1717 %if __YASM_VERSION_ID__ < 0x01030000 && ARCH_X86_64 == 0
1718 %macro vpbroadcastq 2
1719 %if sizeof%1 == 16
1720 movddup %1, %2
1721 %else
1722 vbroadcastsd %1, %2
1723 %endif
1724 %endmacro
1725 %endif
1726%endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t