History

x86: Update x86inc.asm

Make things up-to-date with upstream. https://code.videolan.org/videolan/x86inc.asm

Henrik Gramner committed 2 years ago

Commit afa471d1 parent 782c4df12 files · +519 −270Browse files

Unified · Split

12 files
  1. configure +0 −4
  2. libavcodec/x86/celt_pvq_search.asm +2 −2
  3. libavcodec/x86/h264_chromamc.asm +5 −5
  4. libavcodec/x86/h264_idct.asm +3 −3
  5. libavcodec/x86/h264_intrapred.asm +6 −9
  6. libavcodec/x86/hevc_mc.asm +8 −8
  7. libavcodec/x86/rv40dsp.asm +6 −6
  8. libavcodec/x86/sbrdsp.asm +3 −3
  9. libavcodec/x86/vp8dsp.asm +15 −15
  10. libavcodec/x86/vp9itxfm.asm +2 −0
  11. libavcodec/x86/vp9itxfm_16bpp.asm +7 −5
  12. libavutil/x86/x86inc.asm +462 −210
configure+0−4
@@ -2222,7 +2222,6 @@ ARCH_EXT_LIST_RISCV=" · 2221 unmodified lines
22222222
22232223 ARCH_EXT_LIST_X86="
22242224 $ARCH_EXT_LIST_X86_SIMD
2225− cpunop
22262225 i686
22272226 "
22282227
@@ -2771,7 +2770,6 @@ mipsdsp_deps="mips" · 542 unmodified lines
27712770 mipsdspr2_deps="mips"
27722771 msa_deps="mipsfpu"
27732772
2774−cpunop_deps="i686"
27752773 x86_64_select="i686"
27762774 x86_64_suggest="fast_cmov"
27772775
@@ -6401,7 +6399,6 @@ EOF · 3623 unmodified lines
64016399 done
64026400 disabled x86asm && die "nasm/yasm not found or too old. Use --disable-x86asm for a crippled build."
64036401 X86ASMFLAGS="-f $objformat"
6404− enabled pic && append X86ASMFLAGS "-DPIC"
64056402 test -n "$extern_prefix" && append X86ASMFLAGS "-DPREFIX"
64066403 case "$objformat" in
64076404 elf*) enabled debug && append X86ASMFLAGS $x86asm_debug ;;
@@ -6412,7 +6409,6 @@ EOF · 4 unmodified lines
64126409 enabled avx2 && check_x86asm avx2_external "vextracti128 xmm0, ymm0, 0"
64136410 enabled xop && check_x86asm xop_external "vpmacsdd xmm0, xmm1, xmm2, xmm3"
64146411 enabled fma4 && check_x86asm fma4_external "vfmaddps ymm0, ymm1, ymm2, ymm3"
6415− check_x86asm cpunop "CPU amdnop"
64166412 fi
64176413
64186414 case "$cpu" in
libavcodec/x86/celt_pvq_search.asm+2−2
@@ -74,7 +74,7 @@ SECTION .text · 73 unmodified lines
7474 ; "movaps m0, [r5 + r4]" if PIC is enabled
7575 ; "movaps m0, [constant_name + r4]" if texrel are used
7676 %macro SET_PIC_BASE 3; reg, const_label
77−%ifdef PIC
77+%if PIC
7878 %{1} %2, [%3] ; lea r5, [rip+const]
7979 %define pic_base_%3 %2
8080 %else
@@ -195,7 +195,7 @@ align 16 · 114 unmodified lines
195195 ; PIC relative addressing. Use this
196196 ; to count it in cglobal
197197 ;
198−%ifdef PIC
198+%if PIC
199199 %define num_pic_regs 1
200200 %else
201201 %define num_pic_regs 0
libavcodec/x86/h264_chromamc.asm+5−5
@@ -91,7 +91,7 @@ SECTION .text · 90 unmodified lines
9191
9292 %macro chroma_mc8_mmx_func 2-3
9393 %ifidn %2, rv40
94−%ifdef PIC
94+%if PIC
9595 %define rnd_1d_rv40 r8
9696 %define rnd_2d_rv40 r8
9797 %define extra_regs 2
@@ -147,7 +147,7 @@ cglobal %1_%2_chroma_mc8%3, 6, 7 + extra_regs, 0 · 49 unmodified lines
147147 or r4d, r5d ; x + y
148148
149149 %ifidn %2, rv40
150−%ifdef PIC
150+%if PIC
151151 lea r8, [rnd_rv40_1d_tbl]
152152 %endif
153153 %if ARCH_X86_64 == 0
@@ -198,7 +198,7 @@ cglobal %1_%2_chroma_mc8%3, 6, 7 + extra_regs, 0 · 44 unmodified lines
198198 movd m4, r4d ; x
199199 movd m6, r5d ; y
200200 %ifidn %2, rv40
201−%ifdef PIC
201+%if PIC
202202 lea r8, [rnd_rv40_2d_tbl]
203203 %endif
204204 %if ARCH_X86_64 == 0
@@ -283,7 +283,7 @@ cglobal %1_%2_chroma_mc8%3, 6, 7 + extra_regs, 0 · 78 unmodified lines
283283 %macro chroma_mc4_mmx_func 2
284284 %define extra_regs 0
285285 %ifidn %2, rv40
286−%ifdef PIC
286+%if PIC
287287 %define extra_regs 1
288288 %endif ; PIC
289289 %endif ; rv40
@@ -301,7 +301,7 @@ cglobal %1_%2_chroma_mc4, 6, 6 + extra_regs, 0 · 11 unmodified lines
301301 psubw m5, m3
302302
303303 %ifidn %2, rv40
304−%ifdef PIC
304+%if PIC
305305 lea r6, [rnd_rv40_2d_tbl]
306306 %define rnd_2d_rv40 r6
307307 %else
libavcodec/x86/h264_idct.asm+3−3
@@ -42,7 +42,7 @@ scan8_mem: db 4+ 1*8, 5+ 1*8, 4+ 2*8, 5+ 2*8 · 41 unmodified lines
4242 db 6+11*8, 7+11*8, 6+12*8, 7+12*8
4343 db 4+13*8, 5+13*8, 4+14*8, 5+14*8
4444 db 6+13*8, 7+13*8, 6+14*8, 7+14*8
45−%ifdef PIC
45+%if PIC
4646 %define npicregs 1
4747 %define scan8 picregq
4848 %else
@@ -322,7 +322,7 @@ INIT_XMM sse2 · 273 unmodified lines
322322 cglobal h264_idct8_add4_8, 5, 8 + npicregs, 10, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
323323 movsxdifnidn r3, r3d
324324 xor r5, r5
325−%ifdef PIC
325+%if PIC
326326 lea picregq, [scan8_mem]
327327 %endif
328328 .nextblock:
@@ -398,7 +398,7 @@ h264_idct_add8_mmx_plane: · 69 unmodified lines
398398 cglobal h264_idct_add8_422_8, 5, 8 + npicregs, 0, dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
399399 ; dst1, block_offset, block, stride, nnzc, cntr, coeff, dst2, picreg
400400 movsxdifnidn r3, r3d
401−%ifdef PIC
401+%if PIC
402402 lea picregq, [scan8_mem]
403403 %endif
404404 %if ARCH_X86_64
libavcodec/x86/h264_intrapred.asm+6−9
@@ -1311,10 +1311,7 @@ PRED8x8L_DOWN_RIGHT · 1310 unmodified lines
13111311 ;-----------------------------------------------------------------------------
13121312
13131313 %macro PRED8x8L_VERTICAL_RIGHT 0
1314−cglobal pred8x8l_vertical_right_8, 4,5,7
1315− ; manually spill XMM registers for Win64 because
1316− ; the code here is initialized with INIT_MMX
1317− WIN64_SPILL_XMM 7
1314+cglobal pred8x8l_vertical_right_8, 4,5,6
13181315 sub r0, r3
13191316 lea r4, [r0+r3*2]
13201317 movq mm0, [r0+r3*1-8]
@@ -1384,7 +1381,6 @@ cglobal pred8x8l_vertical_right_8, 4,5,7 · 63 unmodified lines
13841381 movq2dq xmm4, mm6
13851382 pslldq xmm4, 8
13861383 por xmm0, xmm4
1387− movdqa xmm6, [pw_ff00]
13881384 movdqa xmm1, xmm0
13891385 lea r2, [r1+r3*2]
13901386 movdqa xmm2, xmm0
@@ -1394,15 +1390,16 @@ cglobal pred8x8l_vertical_right_8, 4,5,7 · 3 unmodified lines
13941390 pavgb xmm2, xmm0
13951391 INIT_XMM cpuname
13961392 PRED4x4_LOWPASS xmm4, xmm3, xmm1, xmm0, xmm5
1397− pandn xmm6, xmm4
1393+ movdqa xmm0, [pw_ff00]
1394+ pandn xmm0, xmm4
13981395 movdqa xmm5, xmm4
13991396 psrlw xmm4, 8
1400− packuswb xmm6, xmm4
1401− movhlps xmm4, xmm6
1397+ packuswb xmm0, xmm4
1398+ movhlps xmm4, xmm0
14021399 movhps [r0+r3*2], xmm5
14031400 movhps [r0+r3*1], xmm2
14041401 psrldq xmm5, 4
1405− movss xmm5, xmm6
1402+ movss xmm5, xmm0
14061403 psrldq xmm2, 4
14071404 movss xmm2, xmm4
14081405 lea r0, [r2+r3*2]
libavcodec/x86/hevc_mc.asm+8−8
@@ -180,7 +180,7 @@ SECTION .text · 179 unmodified lines
180180 %macro EPEL_FILTER 5 ; bit depth, filter index, xmma, xmmb, gprtmp
181181 %if cpuflag(avx2)
182182 %assign %%offset 32
183−%ifdef PIC
183+%if PIC
184184 lea %5q, [hevc_epel_filters_avx2_%1]
185185 %define FILTER %5q
186186 %else
@@ -188,7 +188,7 @@ SECTION .text · 1 unmodified line
188188 %endif
189189 %else
190190 %assign %%offset 16
191−%ifdef PIC
191+%if PIC
192192 lea %5q, [hevc_epel_filters_sse4_%1]
193193 %define FILTER %5q
194194 %else
@@ -216,7 +216,7 @@ SECTION .text · 21 unmodified lines
216216 %define %%table hevc_epel_filters_sse4_%1
217217 %endif
218218
219−%ifdef PIC
219+%if PIC
220220 lea r3srcq, [%%table]
221221 %define FILTER r3srcq
222222 %else
@@ -234,7 +234,7 @@ SECTION .text · 11 unmodified lines
234234 %else
235235 %define %%table hevc_epel_filters_sse4_10
236236 %endif
237−%ifdef PIC
237+%if PIC
238238 lea r3srcq, [%%table]
239239 %define FILTER r3srcq
240240 %else
@@ -257,7 +257,7 @@ SECTION .text · 16 unmodified lines
257257 %define %%table hevc_qpel_filters_sse4_%1
258258 %endif
259259
260−%ifdef PIC
260+%if PIC
261261 lea rfilterq, [%%table]
262262 %else
263263 %define rfilterq %%table
@@ -576,7 +576,7 @@ SECTION .text · 312 unmodified lines
576576 %define %%table hevc_qpel_filters_sse4_%2
577577 %endif
578578
579−%ifdef PIC
579+%if PIC
580580 lea rfilterq, [%%table]
581581 %else
582582 %define rfilterq %%table
@@ -1288,7 +1288,7 @@ HEVC_PUT_HEVC_QPEL_HV 16, 10 · 705 unmodified lines
12881288 %assign %%offset 4
12891289 dec %2q
12901290 shl %2q, 3
1291−%ifdef PIC
1291+%if PIC
12921292 lea %5q, [%%table]
12931293 %define FILTER %5q
12941294 %else
@@ -1365,7 +1365,7 @@ cglobal hevc_put_hevc_qpel_hv%1_%2, 6, 7, 27, dst, src, srcstride, height, mx, m · 70 unmodified lines
13651365 sub myq, 1
13661366 shl myq, 5
13671367 %define %%table hevc_qpel_filters_avx512icl_v_%1
1368−%ifdef PIC
1368+%if PIC
13691369 lea tmpq, [%%table]
13701370 %define FILTER tmpq
13711371 %else
libavcodec/x86/rv40dsp.asm+6−6
@@ -51,7 +51,7 @@ sixtap_filter_v_m: times 8 dw 1 · 50 unmodified lines
5151 times 8 dw 20
5252 times 8 dw 52
5353
54−%ifdef PIC
54+%if PIC
5555 %define sixtap_filter_hw picregq
5656 %define sixtap_filter_hb picregq
5757 %define sixtap_filter_v picregq
@@ -84,7 +84,7 @@ SECTION .text · 26 unmodified lines
8484 %if WIN64
8585 movsxd %1q, %1d
8686 %endif
87−%ifdef PIC
87+%if PIC
8888 add %1q, picregq
8989 %else
9090 add %1q, %2
@@ -104,7 +104,7 @@ SECTION .text · 13 unmodified lines
104104
105105 %macro FILTER_V 1
106106 cglobal %1_rv40_qpel_v, 6,6+npicregs,12, dst, dststride, src, srcstride, height, my, picreg
107−%ifdef PIC
107+%if PIC
108108 lea picregq, [sixtap_filter_v_m]
109109 %endif
110110 pxor m7, m7
@@ -175,7 +175,7 @@ cglobal %1_rv40_qpel_v, 6,6+npicregs,12, dst, dststride, src, srcstride, height, · 64 unmodified lines
175175
176176 %macro FILTER_H 1
177177 cglobal %1_rv40_qpel_h, 6, 6+npicregs, 12, dst, dststride, src, srcstride, height, mx, picreg
178−%ifdef PIC
178+%if PIC
179179 lea picregq, [sixtap_filter_v_m]
180180 %endif
181181 pxor m7, m7
@@ -238,7 +238,7 @@ FILTER_V avg · 56 unmodified lines
238238
239239 %macro FILTER_SSSE3 1
240240 cglobal %1_rv40_qpel_v, 6,6+npicregs,8, dst, dststride, src, srcstride, height, my, picreg
241−%ifdef PIC
241+%if PIC
242242 lea picregq, [sixtap_filter_hb_m]
243243 %endif
244244
@@ -283,7 +283,7 @@ cglobal %1_rv40_qpel_v, 6,6+npicregs,8, dst, dststride, src, srcstride, height, · 38 unmodified lines
283283 RET
284284
285285 cglobal %1_rv40_qpel_h, 6,6+npicregs,8, dst, dststride, src, srcstride, height, mx, picreg
286−%ifdef PIC
286+%if PIC
287287 lea picregq, [sixtap_filter_hb_m]
288288 %endif
289289 mova m3, [filter_h6_shuf2]
libavcodec/x86/sbrdsp.asm+3−3
@@ -308,7 +308,7 @@ cglobal sbr_qmf_pre_shuffle, 1,4,6,z · 307 unmodified lines
308308 movq [r2q], m2
309309 RET
310310
311−%ifdef PIC
311+%if PIC
312312 %define NREGS 1
313313 %if UNIX64
314314 %define NOISE_TABLE r6q ; r5q is m_max
@@ -321,7 +321,7 @@ cglobal sbr_qmf_pre_shuffle, 1,4,6,z · 6 unmodified lines
321321 %endif
322322
323323 %macro LOAD_NST 1
324−%ifdef PIC
324+%if PIC
325325 lea NOISE_TABLE, [%1]
326326 mova m0, [kxq + NOISE_TABLE]
327327 %else
@@ -371,7 +371,7 @@ apply_noise_main: · 43 unmodified lines
371371 movsxdifnidn noiseq, noised
372372 dec noiseq
373373 shl countd, 2
374−%ifdef PIC
374+%if PIC
375375 lea NOISE_TABLE, [sbr_noise_table]
376376 %endif
377377 lea Yq, [Yq + 2*countq]
libavcodec/x86/vp8dsp.asm+15−15
@@ -114,7 +114,7 @@ bilinear_filter_vb_m: times 8 db 7, 1 · 113 unmodified lines
114114 times 8 db 2, 6
115115 times 8 db 1, 7
116116
117−%ifdef PIC
117+%if PIC
118118 %define fourtap_filter_hw picregq
119119 %define sixtap_filter_hw picregq
120120 %define fourtap_filter_hb picregq
@@ -166,7 +166,7 @@ cglobal put_vp8_epel%1_h6, 6, 6 + npicregs, 8, dst, dststride, src, srcstride, h · 45 unmodified lines
166166 lea mxd, [mxq*3]
167167 mova m3, [filter_h6_shuf2]
168168 mova m4, [filter_h6_shuf3]
169−%ifdef PIC
169+%if PIC
170170 lea picregq, [sixtap_filter_hb_m]
171171 %endif
172172 mova m5, [sixtap_filter_hb+mxq*8-48] ; set up 6tap filter in bytes
@@ -207,7 +207,7 @@ cglobal put_vp8_epel%1_h4, 6, 6 + npicregs, 7, dst, dststride, src, srcstride, h · 34 unmodified lines
207207 mova m2, [pw_256]
208208 mova m3, [filter_h2_shuf]
209209 mova m4, [filter_h4_shuf]
210−%ifdef PIC
210+%if PIC
211211 lea picregq, [fourtap_filter_hb_m]
212212 %endif
213213 mova m5, [fourtap_filter_hb+mxq-16] ; set up 4tap filter in bytes
@@ -234,7 +234,7 @@ cglobal put_vp8_epel%1_h4, 6, 6 + npicregs, 7, dst, dststride, src, srcstride, h · 20 unmodified lines
234234
235235 cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my
236236 shl myd, 4
237−%ifdef PIC
237+%if PIC
238238 lea picregq, [fourtap_filter_hb_m]
239239 %endif
240240 mova m5, [fourtap_filter_hb+myq-16]
@@ -272,7 +272,7 @@ cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picr · 31 unmodified lines
272272
273273 cglobal put_vp8_epel%1_v6, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my
274274 lea myd, [myq*3]
275−%ifdef PIC
275+%if PIC
276276 lea picregq, [sixtap_filter_hb_m]
277277 %endif
278278 lea myq, [sixtap_filter_hb+myq*8]
@@ -326,7 +326,7 @@ FILTER_SSSE3 8 · 47 unmodified lines
326326 INIT_MMX mmxext
327327 cglobal put_vp8_epel4_h4, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, height, mx, picreg
328328 shl mxd, 4
329−%ifdef PIC
329+%if PIC
330330 lea picregq, [fourtap_filter_hw_m]
331331 %endif
332332 movq mm4, [fourtap_filter_hw+mxq-16] ; set up 4tap filter in words
@@ -374,7 +374,7 @@ cglobal put_vp8_epel4_h4, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, he · 41 unmodified lines
374374 INIT_MMX mmxext
375375 cglobal put_vp8_epel4_h6, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, height, mx, picreg
376376 lea mxd, [mxq*3]
377−%ifdef PIC
377+%if PIC
378378 lea picregq, [sixtap_filter_hw_m]
379379 %endif
380380 movq mm4, [sixtap_filter_hw+mxq*8-48] ; set up 4tap filter in words
@@ -431,7 +431,7 @@ cglobal put_vp8_epel4_h6, 6, 6 + npicregs, 0, dst, dststride, src, srcstride, he · 50 unmodified lines
431431 INIT_XMM sse2
432432 cglobal put_vp8_epel8_h4, 6, 6 + npicregs, 10, dst, dststride, src, srcstride, height, mx, picreg
433433 shl mxd, 5
434−%ifdef PIC
434+%if PIC
435435 lea picregq, [fourtap_filter_v_m]
436436 %endif
437437 lea mxq, [fourtap_filter_v+mxq-32]
@@ -480,7 +480,7 @@ INIT_XMM sse2 · 42 unmodified lines
480480 cglobal put_vp8_epel8_h6, 6, 6 + npicregs, 14, dst, dststride, src, srcstride, height, mx, picreg
481481 lea mxd, [mxq*3]
482482 shl mxd, 4
483−%ifdef PIC
483+%if PIC
484484 lea picregq, [sixtap_filter_v_m]
485485 %endif
486486 lea mxq, [sixtap_filter_v+mxq-96]
@@ -543,7 +543,7 @@ cglobal put_vp8_epel8_h6, 6, 6 + npicregs, 14, dst, dststride, src, srcstride, h · 56 unmodified lines
543543 ; 4x4 block, V-only 4-tap filter
544544 cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my
545545 shl myd, 5
546−%ifdef PIC
546+%if PIC
547547 lea picregq, [fourtap_filter_v_m]
548548 %endif
549549 lea myq, [fourtap_filter_v+myq-32]
@@ -597,7 +597,7 @@ cglobal put_vp8_epel%1_v4, 7, 7, 8, dst, dststride, src, srcstride, height, picr · 47 unmodified lines
597597 cglobal put_vp8_epel%1_v6, 7, 7, 8, dst, dststride, src, srcstride, height, picreg, my
598598 shl myd, 4
599599 lea myq, [myq*3]
600−%ifdef PIC
600+%if PIC
601601 lea picregq, [sixtap_filter_v_m]
602602 %endif
603603 lea myq, [sixtap_filter_v+myq-96]
@@ -667,7 +667,7 @@ FILTER_V 8 · 63 unmodified lines
667667 %if cpuflag(ssse3)
668668 cglobal put_vp8_bilinear%1_v, 7, 7, 5, dst, dststride, src, srcstride, height, picreg, my
669669 shl myd, 4
670−%ifdef PIC
670+%if PIC
671671 lea picregq, [bilinear_filter_vb_m]
672672 %endif
673673 pxor m4, m4
@@ -697,7 +697,7 @@ cglobal put_vp8_bilinear%1_v, 7, 7, 5, dst, dststride, src, srcstride, height, p · 23 unmodified lines
697697 %else ; cpuflag(ssse3)
698698 cglobal put_vp8_bilinear%1_v, 7, 7, 7, dst, dststride, src, srcstride, height, picreg, my
699699 shl myd, 4
700−%ifdef PIC
700+%if PIC
701701 lea picregq, [bilinear_filter_vw_m]
702702 %endif
703703 pxor m6, m6
@@ -743,7 +743,7 @@ cglobal put_vp8_bilinear%1_v, 7, 7, 7, dst, dststride, src, srcstride, height, p · 39 unmodified lines
743743 %if cpuflag(ssse3)
744744 cglobal put_vp8_bilinear%1_h, 6, 6 + npicregs, 5, dst, dststride, src, srcstride, height, mx, picreg
745745 shl mxd, 4
746−%ifdef PIC
746+%if PIC
747747 lea picregq, [bilinear_filter_vb_m]
748748 %endif
749749 pxor m4, m4
@@ -773,7 +773,7 @@ cglobal put_vp8_bilinear%1_h, 6, 6 + npicregs, 5, dst, dststride, src, srcstride · 23 unmodified lines
773773 %else ; cpuflag(ssse3)
774774 cglobal put_vp8_bilinear%1_h, 6, 6 + npicregs, 7, dst, dststride, src, srcstride, height, mx, picreg
775775 shl mxd, 4
776−%ifdef PIC
776+%if PIC
777777 lea picregq, [bilinear_filter_vw_m]
778778 %endif
779779 pxor m6, m6
libavcodec/x86/vp9itxfm.asm+2−0
@@ -330,7 +330,9 @@ IDCT_4x4_FN ssse3 · 329 unmodified lines
330330 INIT_MMX %5
331331 cglobal vp9_%1_%3_4x4_add, 3, 3, 0, dst, stride, block, eob
332332 %if WIN64 && notcpuflag(ssse3)
333+INIT_XMM cpuname
333334 WIN64_SPILL_XMM 8
335+INIT_MMX cpuname
334336 %endif
335337 movdqa xmm5, [pd_8192]
336338 mova m0, [blockq+ 0]
libavcodec/x86/vp9itxfm_16bpp.asm+7−5
@@ -303,7 +303,9 @@ IDCT4_10_FN · 302 unmodified lines
303303 %macro IADST4_FN 4
304304 cglobal vp9_%1_%3_4x4_add_10, 3, 3, 0, dst, stride, block, eob
305305 %if WIN64 && notcpuflag(ssse3)
306+INIT_XMM cpuname
306307 WIN64_SPILL_XMM 8
308+INIT_MMX cpuname
307309 %endif
308310 movdqa xmm5, [pd_8192]
309311 mova m0, [blockq+0*16+0]
@@ -672,7 +674,7 @@ cglobal vp9_idct_idct_8x8_add_10, 4, 6 + ARCH_X86_64, 14, \ · 362 unmodified lines
672674 mov dstbakq, dstq
673675 movsxd cntq, cntd
674676 %endif
675−%ifdef PIC
677+%if PIC
676678 lea ptrq, [default_8x8]
677679 movzx cntd, byte [ptrq+cntq-1]
678680 %else
@@ -921,7 +923,7 @@ cglobal vp9_%1_%3_8x8_add_10, 4, 6 + ARCH_X86_64, 16, \ · 242 unmodified lines
921923 mov dstbakq, dstq
922924 movsxd cntq, cntd
923925 %endif
924−%ifdef PIC
926+%if PIC
925927 lea ptrq, [%5_8x8]
926928 movzx cntd, byte [ptrq+cntq-1]
927929 %else
@@ -1128,7 +1130,7 @@ cglobal vp9_idct_idct_16x16_add_10, 4, 6 + ARCH_X86_64, 16, \ · 200 unmodified lines
11281130 mov dstbakq, dstq
11291131 movsxd cntq, cntd
11301132 %endif
1131−%ifdef PIC
1133+%if PIC
11321134 lea ptrq, [default_16x16]
11331135 movzx cntd, byte [ptrq+cntq-1]
11341136 %else
@@ -1445,7 +1447,7 @@ cglobal vp9_%1_%4_16x16_add_10, 4, 6 + ARCH_X86_64, 16, \ · 310 unmodified lines
14451447 mov dstbakq, dstq
14461448 movsxd cntq, cntd
14471449 %endif
1448−%ifdef PIC
1450+%if PIC
14491451 lea ptrq, [%7_16x16]
14501452 movzx cntd, byte [ptrq+cntq-1]
14511453 %else
@@ -1958,7 +1960,7 @@ cglobal vp9_idct_idct_32x32_add_10, 4, 6 + ARCH_X86_64, 16, \ · 506 unmodified lines
19581960 mov dstbakq, dstq
19591961 movsxd cntq, cntd
19601962 %endif
1961−%ifdef PIC
1963+%if PIC
19621964 lea ptrq, [default_32x32]
19631965 movzx cntd, byte [ptrq+cntq-1]
19641966 %else
libavutil/x86/x86inc.asm+462−210
@@ -1,7 +1,7 @@
11 ;*****************************************************************************
2−;* x86inc.asm: x264asm abstraction layer
2+;* x86inc.asm: x86 abstraction layer
33 ;*****************************************************************************
4−;* Copyright (C) 2005-2018 x264 project
4+;* Copyright (C) 2005-2024 x264 project
55 ;*
66 ;* Authors: Loren Merritt <lorenm@u.washington.edu>
77 ;* Henrik Gramner <henrik@gramner.com>
@@ -21,21 +21,14 @@ · 13 unmodified lines
2121 ;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
2222 ;*****************************************************************************
2323
24−; This is a header file for the x264ASM assembly language, which uses
24+; This is a header file for the x86inc.asm assembly language, which uses
2525 ; NASM/YASM syntax combined with a large number of macros to provide easy
2626 ; abstraction between different calling conventions (x86_32, win64, linux64).
2727 ; It also has various other useful features to simplify writing the kind of
28−; DSP functions that are most often used in x264.
29−
30−; Unlike the rest of x264, this file is available under an ISC license, as it
31−; has significant usefulness outside of x264 and we want it to be available
32−; to the largest audience possible. Of course, if you modify it for your own
33−; purposes to add a new feature, we strongly encourage contributing a patch
34−; as this feature might be useful for others as well. Send patches or ideas
35−; to x264-devel@videolan.org .
28+; DSP functions that are most often used.
3629
3730 %ifndef private_prefix
38− %define private_prefix x264
31+ %error private_prefix not defined
3932 %endif
4033
4134 %ifndef public_prefix
@@ -68,12 +61,19 @@ · 26 unmodified lines
6861 %endif
6962
7063 %define FORMAT_ELF 0
64+%define FORMAT_MACHO 0
7165 %ifidn __OUTPUT_FORMAT__,elf
7266 %define FORMAT_ELF 1
7367 %elifidn __OUTPUT_FORMAT__,elf32
7468 %define FORMAT_ELF 1
7569 %elifidn __OUTPUT_FORMAT__,elf64
7670 %define FORMAT_ELF 1
71+%elifidn __OUTPUT_FORMAT__,macho
72+ %define FORMAT_MACHO 1
73+%elifidn __OUTPUT_FORMAT__,macho32
74+ %define FORMAT_MACHO 1
75+%elifidn __OUTPUT_FORMAT__,macho64
76+ %define FORMAT_MACHO 1
7777 %endif
7878
7979 %ifdef PREFIX
@@ -82,6 +82,11 @@ · 2 unmodified lines
8282 %define mangle(x) x
8383 %endif
8484
85+; Use VEX-encoding even in non-AVX functions
86+%ifndef FORCE_VEX_ENCODING
87+ %define FORCE_VEX_ENCODING 0
88+%endif
89+
8590 ; aout does not support align=
8691 ; NOTE: This section is out of sync with x264, in order to
8792 ; keep supporting OS/2.
@@ -99,28 +104,27 @@ · 11 unmodified lines
99104 %endif
100105 %endmacro
101106
102−%if WIN64
103− %define PIC
104−%elif ARCH_X86_64 == 0
105−; x86_32 doesn't require PIC.
106−; Some distros prefer shared objects to be PIC, but nothing breaks if
107−; the code contains a few textrels, so we'll skip that complexity.
108− %undef PIC
109−%endif
110−%ifdef PIC
107+%if ARCH_X86_64
108+ %define PIC 1 ; always use PIC on x86-64
111109 default rel
110+%elifidn __OUTPUT_FORMAT__,win32
111+ %define PIC 0 ; PIC isn't used on 32-bit Windows
112+%elifndef PIC
113+ %define PIC 0
112114 %endif
113115
114−%macro CPUNOP 1
115− %if HAVE_CPUNOP
116− CPU %1
116+%define HAVE_PRIVATE_EXTERN 1
117+%ifdef __NASM_VERSION_ID__
118+ %use smartalign
119+ %if __NASM_VERSION_ID__ < 0x020e0000 ; 2.14
120+ %define HAVE_PRIVATE_EXTERN 0
117121 %endif
118−%endmacro
122+%endif
119123
120124 ; Macros to eliminate most code duplication between x86_32 and x86_64:
121125 ; Currently this works only for leaf functions which load all their arguments
122126 ; into registers at the start, and make no other use of the stack. Luckily that
123−; covers most of x264's asm.
127+; covers most use cases.
124128
125129 ; PROLOGUE:
126130 ; %1 = number of arguments. loads them from stack if needed.
@@ -232,6 +236,18 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 105 unmodified lines
232236 %define gprsize 4
233237 %endif
234238
239+%macro LEA 2
240+%if ARCH_X86_64
241+ lea %1, [%2]
242+%elif PIC
243+ call $+5 ; special-cased to not affect the RSB on most CPU:s
244+ pop %1
245+ add %1, (%2)-$+1
246+%else
247+ mov %1, %2
248+%endif
249+%endmacro
250+
235251 ; Repeats an instruction/operation for multiple arguments.
236252 ; Example usage: "REPX {psrlw x, 8}, m0, m1, m2, m3"
237253 %macro REPX 2-* ; operation, args
@@ -303,6 +319,10 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 65 unmodified lines
303319 %endif
304320 %endmacro
305321
322+%if ARCH_X86_64 == 0
323+ %define movsxd movifnidn
324+%endif
325+
306326 %macro movsxdifnidn 2
307327 %ifnidn %1, %2
308328 movsxd %1, %2
@@ -354,7 +374,46 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 45 unmodified lines
354374 %define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512)))
355375 %define high_mm_regs (16*cpuflag(avx512))
356376
357−%macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only)
377+; Large stack allocations on Windows need to use stack probing in order
378+; to guarantee that all stack memory is committed before accessing it.
379+; This is done by ensuring that the guard page(s) at the end of the
380+; currently committed pages are touched prior to any pages beyond that.
381+%if WIN64
382+ %assign STACK_PROBE_SIZE 8192
383+%elifidn __OUTPUT_FORMAT__, win32
384+ %assign STACK_PROBE_SIZE 4096
385+%else
386+ %assign STACK_PROBE_SIZE 0
387+%endif
388+
389+%macro PROBE_STACK 1 ; stack_size
390+ %if STACK_PROBE_SIZE
391+ %assign %%i STACK_PROBE_SIZE
392+ %rep %1 / STACK_PROBE_SIZE
393+ mov eax, [rsp-%%i]
394+ %assign %%i %%i+STACK_PROBE_SIZE
395+ %endrep
396+ %endif
397+%endmacro
398+
399+%macro RESET_STACK_STATE 0
400+ %ifidn rstk, rsp
401+ %assign stack_offset stack_offset - stack_size_padded
402+ %else
403+ %xdefine rstk rsp
404+ %endif
405+ %assign stack_size 0
406+ %assign stack_size_padded 0
407+ %assign xmm_regs_used 0
408+%endmacro
409+
410+%macro ALLOC_STACK 0-2 0, 0 ; stack_size, n_xmm_regs
411+ RESET_STACK_STATE
412+ %ifnum %2
413+ %if mmsize != 8
414+ %assign xmm_regs_used %2
415+ %endif
416+ %endif
358417 %ifnum %1
359418 %if %1 != 0
360419 %assign %%pad 0
@@ -364,16 +423,14 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 3 unmodified lines
364423 %endif
365424 %if WIN64
366425 %assign %%pad %%pad + 32 ; shadow space
367− %if mmsize != 8
368− %assign xmm_regs_used %2
369− %if xmm_regs_used > 8
370− %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers
371− %endif
426+ %if xmm_regs_used > 8
427+ %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers
372428 %endif
373429 %endif
374430 %if required_stack_alignment <= STACK_ALIGNMENT
375431 ; maintain the current stack alignment
376432 %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
433+ PROBE_STACK stack_size_padded
377434 SUB rsp, stack_size_padded
378435 %else
379436 %assign %%reg_num (regs_used - 1)
@@ -389,6 +446,7 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 9 unmodified lines
389446 %xdefine rstkm rstk
390447 %endif
391448 %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1))
449+ PROBE_STACK stack_size_padded
392450 mov rstk, rsp
393451 and rsp, ~(required_stack_alignment-1)
394452 sub rsp, stack_size_padded
@@ -399,7 +457,7 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 4 unmodified lines
399457 %endif
400458 %endmacro
401459
402−%macro SETUP_STACK_POINTER 1
460+%macro SETUP_STACK_POINTER 0-1 0
403461 %ifnum %1
404462 %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT
405463 %if %1 > 0
@@ -462,35 +520,62 @@ DECLARE_REG 14, R13, 120 · 56 unmodified lines
462520 %endif
463521 %endmacro
464522
465−%macro WIN64_PUSH_XMM 0
466− ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated.
467− %if xmm_regs_used > 6 + high_mm_regs
468− movaps [rstk + stack_offset + 8], xmm6
469− %endif
470− %if xmm_regs_used > 7 + high_mm_regs
471− movaps [rstk + stack_offset + 24], xmm7
472− %endif
473− %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8
474− %if %%xmm_regs_on_stack > 0
475− %assign %%i 8
476− %rep %%xmm_regs_on_stack
477− movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i
478− %assign %%i %%i+1
479− %endrep
523+; Push XMM registers to the stack. If no argument is specified all used register
524+; will be pushed, otherwise only push previously unpushed registers.
525+%macro WIN64_PUSH_XMM 0-2 ; new_xmm_regs_used, xmm_regs_pushed
526+ %if mmsize != 8
527+ %if %0 == 2
528+ %assign %%pushed %2
529+ %assign xmm_regs_used %1
530+ %elif %0 == 1
531+ %assign %%pushed xmm_regs_used
532+ %assign xmm_regs_used %1
533+ %else
534+ %assign %%pushed 0
535+ %endif
536+ ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated.
537+ %if %%pushed <= 6 + high_mm_regs && xmm_regs_used > 6 + high_mm_regs
538+ movaps [rstk + stack_offset + 8], xmm6
539+ %endif
540+ %if %%pushed <= 7 + high_mm_regs && xmm_regs_used > 7 + high_mm_regs
541+ movaps [rstk + stack_offset + 24], xmm7
542+ %endif
543+ %assign %%pushed %%pushed - high_mm_regs - 8
544+ %if %%pushed < 0
545+ %assign %%pushed 0
546+ %endif
547+ %assign %%regs_to_push xmm_regs_used - %%pushed - high_mm_regs - 8
548+ %if %%regs_to_push > 0
549+ ASSERT (%%regs_to_push + %%pushed) * 16 <= stack_size_padded - stack_size - 32
550+ %assign %%i %%pushed + 8
551+ %rep %%regs_to_push
552+ movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i
553+ %assign %%i %%i+1
554+ %endrep
555+ %endif
480556 %endif
481557 %endmacro
482558
483−%macro WIN64_SPILL_XMM 1
484− %assign xmm_regs_used %1
485− ASSERT xmm_regs_used <= 16 + high_mm_regs
486− %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8
487− %if %%xmm_regs_on_stack > 0
488− ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack.
489− %assign %%pad %%xmm_regs_on_stack*16 + 32
490− %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
491− SUB rsp, stack_size_padded
559+; Allocated stack space for XMM registers and push all, or a subset, of those
560+%macro WIN64_SPILL_XMM 1-2 ; xmm_regs_used, xmm_regs_reserved
561+ RESET_STACK_STATE
562+ %if mmsize != 8
563+ %assign xmm_regs_used %1
564+ ASSERT xmm_regs_used <= 16 + high_mm_regs
565+ %if %0 == 2
566+ ASSERT %2 >= %1
567+ %assign %%xmm_regs_on_stack %2 - high_mm_regs - 8
568+ %else
569+ %assign %%xmm_regs_on_stack %1 - high_mm_regs - 8
570+ %endif
571+ %if %%xmm_regs_on_stack > 0
572+ ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack.
573+ %assign %%pad %%xmm_regs_on_stack*16 + 32
574+ %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1))
575+ SUB rsp, stack_size_padded
576+ %endif
577+ WIN64_PUSH_XMM
492578 %endif
493− WIN64_PUSH_XMM
494579 %endmacro
495580
496581 %macro WIN64_RESTORE_XMM_INTERNAL 0
@@ -521,9 +606,7 @@ DECLARE_REG 14, R13, 120 · 24 unmodified lines
521606
522607 %macro WIN64_RESTORE_XMM 0
523608 WIN64_RESTORE_XMM_INTERNAL
524− %assign stack_offset (stack_offset-stack_size_padded)
525− %assign stack_size_padded 0
526− %assign xmm_regs_used 0
609+ RESET_STACK_STATE
527610 %endmacro
528611
529612 %define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs
@@ -558,12 +641,11 @@ DECLARE_REG 14, R13, 72 · 28 unmodified lines
558641 %macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names...
559642 %assign num_args %1
560643 %assign regs_used %2
561− %assign xmm_regs_used %3
562644 ASSERT regs_used >= num_args
563645 SETUP_STACK_POINTER %4
564646 ASSERT regs_used <= 15
565647 PUSH_IF_USED 9, 10, 11, 12, 13, 14
566− ALLOC_STACK %4
648+ ALLOC_STACK %4, %3
567649 LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14
568650 %if %0 > 4
569651 %ifnum %4
@@ -627,7 +709,7 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 57 unmodified lines
627709 SETUP_STACK_POINTER %4
628710 ASSERT regs_used <= 7
629711 PUSH_IF_USED 3, 4, 5, 6
630− ALLOC_STACK %4
712+ ALLOC_STACK %4, %3
631713 LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6
632714 %if %0 > 4
633715 %ifnum %4
@@ -660,11 +742,19 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 26 unmodified lines
660742 %endif ;======================================================================
661743
662744 %if WIN64 == 0
663− %macro WIN64_SPILL_XMM 1
745+ %macro WIN64_SPILL_XMM 1-2
746+ RESET_STACK_STATE
747+ %if mmsize != 8
748+ %assign xmm_regs_used %1
749+ %endif
664750 %endmacro
665751 %macro WIN64_RESTORE_XMM 0
752+ RESET_STACK_STATE
666753 %endmacro
667− %macro WIN64_PUSH_XMM 0
754+ %macro WIN64_PUSH_XMM 0-2
755+ %if mmsize != 8 && %0 >= 1
756+ %assign xmm_regs_used %1
757+ %endif
668758 %endmacro
669759 %endif
670760
@@ -705,7 +795,7 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 34 unmodified lines
705795
706796 BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp
707797
708−%macro TAIL_CALL 2 ; callee, is_nonadjacent
798+%macro TAIL_CALL 1-2 1 ; callee, is_nonadjacent
709799 %if has_epilogue
710800 call %1
711801 RET
@@ -735,22 +825,25 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 23 unmodified lines
735825 %endmacro
736826 %macro cglobal_internal 2-3+
737827 annotate_function_size
738− %if %1
739− %xdefine %%FUNCTION_PREFIX private_prefix
740− %xdefine %%VISIBILITY hidden
741− %else
742− %xdefine %%FUNCTION_PREFIX public_prefix
743− %xdefine %%VISIBILITY
744− %endif
745828 %ifndef cglobaled_%2
746− %xdefine %2 mangle(%%FUNCTION_PREFIX %+ _ %+ %2)
829+ %if %1
830+ %xdefine %2 mangle(private_prefix %+ _ %+ %2)
831+ %else
832+ %xdefine %2 mangle(public_prefix %+ _ %+ %2)
833+ %endif
747834 %xdefine %2.skip_prologue %2 %+ .skip_prologue
748835 CAT_XDEFINE cglobaled_, %2, 1
749836 %endif
750837 %xdefine current_function %2
751838 %xdefine current_function_section __SECT__
752839 %if FORMAT_ELF
753− global %2:function %%VISIBILITY
840+ %if %1
841+ global %2:function hidden
842+ %else
843+ global %2:function
844+ %endif
845+ %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN && %1
846+ global %2:private_extern
754847 %else
755848 global %2
756849 %endif
@@ -771,6 +864,8 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 14 unmodified lines
771864 %macro cglobal_label 1
772865 %if FORMAT_ELF
773866 global current_function %+ %1:function hidden
867+ %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN
868+ global current_function %+ %1:private_extern
774869 %else
775870 global current_function %+ %1
776871 %endif
@@ -796,15 +891,34 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 19 unmodified lines
796891 %xdefine %1 mangle(private_prefix %+ _ %+ %1)
797892 %if FORMAT_ELF
798893 global %1:data hidden
894+ %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN
895+ global %1:private_extern
799896 %else
800897 global %1
801898 %endif
802899 %1: %2
803900 %endmacro
804901
805−; This is needed for ELF, otherwise the GNU linker assumes the stack is executable by default.
806902 %if FORMAT_ELF
903+ ; The GNU linker assumes the stack is executable by default.
807904 [SECTION .note.GNU-stack noalloc noexec nowrite progbits]
905+
906+ %ifdef __NASM_VERSION_ID__
907+ %if __NASM_VERSION_ID__ >= 0x020e0300 ; 2.14.03
908+ %if ARCH_X86_64
909+ ; Control-flow Enforcement Technology (CET) properties.
910+ [SECTION .note.gnu.property alloc noexec nowrite note align=gprsize]
911+ dd 0x00000004 ; n_namesz
912+ dd gprsize + 8 ; n_descsz
913+ dd 0x00000005 ; n_type = NT_GNU_PROPERTY_TYPE_0
914+ db "GNU",0 ; n_name
915+ dd 0xc0000002 ; pr_type = GNU_PROPERTY_X86_FEATURE_1_AND
916+ dd 0x00000004 ; pr_datasz
917+ dd 0x00000002 ; pr_data = GNU_PROPERTY_X86_FEATURE_1_SHSTK
918+ dd 0x00000000 ; pr_padding
919+ %endif
920+ %endif
921+ %endif
808922 %endif
809923
810924 ; Tell debuggers how large the function was.
@@ -828,32 +942,34 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 17 unmodified lines
828942 ; cpuflags
829943
830944 %assign cpuflags_mmx (1<<0)
831−%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx
832−%assign cpuflags_3dnow (1<<2) | cpuflags_mmx
833−%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow
834−%assign cpuflags_sse (1<<4) | cpuflags_mmx2
835−%assign cpuflags_sse2 (1<<5) | cpuflags_sse
836−%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2
837−%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2
838−%assign cpuflags_sse3 (1<<8) | cpuflags_sse2
839−%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3
840−%assign cpuflags_sse4 (1<<10)| cpuflags_ssse3
841−%assign cpuflags_sse42 (1<<11)| cpuflags_sse4
842−%assign cpuflags_aesni (1<<12)| cpuflags_sse42
843−%assign cpuflags_avx (1<<13)| cpuflags_sse42
844−%assign cpuflags_xop (1<<14)| cpuflags_avx
845−%assign cpuflags_fma4 (1<<15)| cpuflags_avx
846−%assign cpuflags_fma3 (1<<16)| cpuflags_avx
847−%assign cpuflags_bmi1 (1<<17)| cpuflags_avx|cpuflags_lzcnt
848−%assign cpuflags_bmi2 (1<<18)| cpuflags_bmi1
849−%assign cpuflags_avx2 (1<<19)| cpuflags_fma3|cpuflags_bmi2
850−%assign cpuflags_avx512 (1<<20)| cpuflags_avx2 ; F, CD, BW, DQ, VL
851−%assign cpuflags_avx512icl (1<<25)| cpuflags_avx512
852−
853−%assign cpuflags_cache32 (1<<21)
854−%assign cpuflags_cache64 (1<<22)
855−%assign cpuflags_aligned (1<<23) ; not a cpu feature, but a function variant
856−%assign cpuflags_atom (1<<24)
945+%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx
946+%assign cpuflags_3dnow (1<<2) | cpuflags_mmx
947+%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow
948+%assign cpuflags_sse (1<<4) | cpuflags_mmx2
949+%assign cpuflags_sse2 (1<<5) | cpuflags_sse
950+%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2
951+%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2
952+%assign cpuflags_sse3 (1<<8) | cpuflags_sse2
953+%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3
954+%assign cpuflags_sse4 (1<<10) | cpuflags_ssse3
955+%assign cpuflags_sse42 (1<<11) | cpuflags_sse4
956+%assign cpuflags_aesni (1<<12) | cpuflags_sse42
957+%assign cpuflags_clmul (1<<13) | cpuflags_sse42
958+%assign cpuflags_gfni (1<<14) | cpuflags_aesni|cpuflags_clmul
959+%assign cpuflags_avx (1<<15) | cpuflags_sse42
960+%assign cpuflags_xop (1<<16) | cpuflags_avx
961+%assign cpuflags_fma4 (1<<17) | cpuflags_avx
962+%assign cpuflags_fma3 (1<<18) | cpuflags_avx
963+%assign cpuflags_bmi1 (1<<19) | cpuflags_avx|cpuflags_lzcnt
964+%assign cpuflags_bmi2 (1<<20) | cpuflags_bmi1
965+%assign cpuflags_avx2 (1<<21) | cpuflags_fma3|cpuflags_bmi2
966+%assign cpuflags_avx512 (1<<22) | cpuflags_avx2 ; F, CD, BW, DQ, VL
967+%assign cpuflags_avx512icl (1<<23) | cpuflags_avx512|cpuflags_gfni ; VNNI, IFMA, VBMI, VBMI2, VPOPCNTDQ, BITALG, VAES, VPCLMULQDQ
968+
969+%assign cpuflags_cache32 (1<<24)
970+%assign cpuflags_cache64 (1<<25)
971+%assign cpuflags_aligned (1<<26) ; not a cpu feature, but a function variant
972+%assign cpuflags_atom (1<<27)
857973
858974 ; Returns a boolean value expressing whether or not the specified cpuflag is enabled.
859975 %define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1)
@@ -895,9 +1011,17 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 35 unmodified lines
8951011 %endif
8961012
8971013 %if ARCH_X86_64 || cpuflag(sse2)
898− CPUNOP amdnop
1014+ %ifdef __NASM_VERSION_ID__
1015+ ALIGNMODE p6
1016+ %else
1017+ CPU amdnop
1018+ %endif
8991019 %else
900− CPUNOP basicnop
1020+ %ifdef __NASM_VERSION_ID__
1021+ ALIGNMODE nop
1022+ %else
1023+ CPU basicnop
1024+ %endif
9011025 %endif
9021026 %endmacro
9031027
@@ -971,7 +1095,7 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 67 unmodified lines
9711095 %endmacro
9721096
9731097 %macro INIT_XMM 0-1+
974− %assign avx_enabled 0
1098+ %assign avx_enabled FORCE_VEX_ENCODING
9751099 %define RESET_MM_PERMUTATION INIT_XMM %1
9761100 %define mmsize 16
9771101 %define mova movdqa
@@ -983,6 +1107,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 5 unmodified lines
9831107 %if WIN64
9841108 AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers
9851109 %endif
1110+ %xdefine bcstw 1to8
1111+ %xdefine bcstd 1to4
1112+ %xdefine bcstq 1to2
9861113 %endmacro
9871114
9881115 %macro INIT_YMM 0-1+
@@ -996,6 +1123,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 7 unmodified lines
9961123 INIT_CPUFLAGS %1
9971124 DEFINE_MMREGS ymm
9981125 AVX512_MM_PERMUTATION
1126+ %xdefine bcstw 1to16
1127+ %xdefine bcstd 1to8
1128+ %xdefine bcstq 1to4
9991129 %endmacro
10001130
10011131 %macro INIT_ZMM 0-1+
@@ -1009,6 +1139,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 7 unmodified lines
10091139 INIT_CPUFLAGS %1
10101140 DEFINE_MMREGS zmm
10111141 AVX512_MM_PERMUTATION
1142+ %xdefine bcstw 1to32
1143+ %xdefine bcstd 1to16
1144+ %xdefine bcstq 1to8
10121145 %endmacro
10131146
10141147 INIT_XMM
@@ -1106,19 +1239,32 @@ INIT_XMM · 91 unmodified lines
11061239 %endif
11071240 %assign %%i 0
11081241 %rep num_mmregs
1109− CAT_XDEFINE %%f, %%i, m %+ %%i
1242+ %xdefine %%tmp m %+ %%i
1243+ CAT_XDEFINE %%f, %%i, regnumof %+ %%tmp
11101244 %assign %%i %%i+1
11111245 %endrep
11121246 %endmacro
11131247
1114−%macro LOAD_MM_PERMUTATION 1 ; name to load from
1115− %ifdef %1_m0
1248+%macro LOAD_MM_PERMUTATION 0-1 ; name to load from
1249+ %if %0
1250+ %xdefine %%f %1_m
1251+ %else
1252+ %xdefine %%f current_function %+ _m
1253+ %endif
1254+ %xdefine %%tmp %%f %+ 0
1255+ %ifnum %%tmp
1256+ DEFINE_MMREGS mmtype
11161257 %assign %%i 0
11171258 %rep num_mmregs
1118− CAT_XDEFINE m, %%i, %1_m %+ %%i
1119− CAT_XDEFINE nn, m %+ %%i, %%i
1259+ %xdefine %%tmp %%f %+ %%i
1260+ CAT_XDEFINE %%m, %%i, m %+ %%tmp
11201261 %assign %%i %%i+1
11211262 %endrep
1263+ %rep num_mmregs
1264+ %assign %%i %%i-1
1265+ CAT_XDEFINE m, %%i, %%m %+ %%i
1266+ CAT_XDEFINE nn, m %+ %%i, %%i
1267+ %endrep
11221268 %endif
11231269 %endmacro
11241270
@@ -1224,8 +1370,22 @@ INIT_XMM · 99 unmodified lines
12241370 %ifdef cpuname
12251371 %if notcpuflag(%2)
12261372 %error use of ``%1'' %2 instruction in cpuname function: current_function
1227− %elif cpuflags_%2 < cpuflags_sse && notcpuflag(sse2) && __sizeofreg > 8
1373+ %elif %3 == 0 && __sizeofreg == 16 && notcpuflag(sse2)
12281374 %error use of ``%1'' sse2 instruction in cpuname function: current_function
1375+ %elif %3 == 0 && __sizeofreg == 32 && notcpuflag(avx2)
1376+ %error use of ``%1'' avx2 instruction in cpuname function: current_function
1377+ %elif __sizeofreg == 16 && notcpuflag(sse)
1378+ %error use of ``%1'' sse instruction in cpuname function: current_function
1379+ %elif __sizeofreg == 32 && notcpuflag(avx)
1380+ %error use of ``%1'' avx instruction in cpuname function: current_function
1381+ %elif __sizeofreg == 64 && notcpuflag(avx512)
1382+ %error use of ``%1'' avx512 instruction in cpuname function: current_function
1383+ %elifidn %1, pextrw ; special case because the base instruction is mmx2,
1384+ %ifnid %6 ; but sse4 is required for memory operands
1385+ %if notcpuflag(sse4)
1386+ %error use of ``%1'' sse4 instruction in cpuname function: current_function
1387+ %endif
1388+ %endif
12291389 %endif
12301390 %endif
12311391 %endif
@@ -1267,11 +1427,79 @@ INIT_XMM · 35 unmodified lines
12671427 %1 %6, __src2
12681428 %endif
12691429 %elif %0 >= 9
1270− __instr %6, %7, %8, %9
1430+ %if avx_enabled && __sizeofreg >= 16 && %4 == 1
1431+ %ifnnum regnumof%7
1432+ %if %3
1433+ vmovaps %6, %7
1434+ %else
1435+ vmovdqa %6, %7
1436+ %endif
1437+ __instr %6, %6, %8, %9
1438+ %else
1439+ __instr %6, %7, %8, %9
1440+ %endif
1441+ %else
1442+ __instr %6, %7, %8, %9
1443+ %endif
12711444 %elif %0 == 8
1272− __instr %6, %7, %8
1445+ %if avx_enabled && __sizeofreg >= 16 && %4 == 0
1446+ %xdefine __src1 %7
1447+ %xdefine __src2 %8
1448+ %if %5
1449+ %ifnum regnumof%7
1450+ %ifnum regnumof%8
1451+ %if regnumof%7 < 8 && regnumof%8 >= 8 && regnumof%8 < 16 && sizeof%8 <= 32
1452+ ; Most VEX-encoded instructions require an additional byte to encode when
1453+ ; src2 is a high register (e.g. m8..15). If the instruction is commutative
1454+ ; we can swap src1 and src2 when doing so reduces the instruction length.
1455+ %xdefine __src1 %8
1456+ %xdefine __src2 %7
1457+ %endif
1458+ %endif
1459+ %elifnum regnumof%8 ; put memory operands in src2 when possible
1460+ %xdefine __src1 %8
1461+ %xdefine __src2 %7
1462+ %else
1463+ %assign __emulate_avx 1
1464+ %endif
1465+ %elifnnum regnumof%7
1466+ ; EVEX allows imm8 shift instructions to be used with memory operands,
1467+ ; but VEX does not. This handles those special cases.
1468+ %ifnnum %8
1469+ %assign __emulate_avx 1
1470+ %elif notcpuflag(avx512)
1471+ %assign __emulate_avx 1
1472+ %endif
1473+ %endif
1474+ %if __emulate_avx ; a separate load is required
1475+ %if %3
1476+ vmovaps %6, %7
1477+ %else
1478+ vmovdqa %6, %7
1479+ %endif
1480+ __instr %6, %6, %8
1481+ %else
1482+ __instr %6, __src1, __src2
1483+ %endif
1484+ %else
1485+ __instr %6, %7, %8
1486+ %endif
12731487 %elif %0 == 7
1274− __instr %6, %7
1488+ %if avx_enabled && __sizeofreg >= 16 && %5
1489+ %xdefine __src1 %6
1490+ %xdefine __src2 %7
1491+ %ifnum regnumof%6
1492+ %ifnum regnumof%7
1493+ %if regnumof%6 < 8 && regnumof%7 >= 8 && regnumof%7 < 16 && sizeof%7 <= 32
1494+ %xdefine __src1 %7
1495+ %xdefine __src2 %6
1496+ %endif
1497+ %endif
1498+ %endif
1499+ __instr %6, __src1, __src2
1500+ %else
1501+ __instr %6, %7
1502+ %endif
12751503 %else
12761504 __instr %6
12771505 %endif
@@ -1318,8 +1546,8 @@ AVX_INSTR andpd, sse2, 1, 0, 1 · 40 unmodified lines
13181546 AVX_INSTR andps, sse, 1, 0, 1
13191547 AVX_INSTR blendpd, sse4, 1, 1, 0
13201548 AVX_INSTR blendps, sse4, 1, 1, 0
1321−AVX_INSTR blendvpd, sse4 ; can't be emulated
1322−AVX_INSTR blendvps, sse4 ; can't be emulated
1549+AVX_INSTR blendvpd, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding
1550+AVX_INSTR blendvps, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding
13231551 AVX_INSTR cmpeqpd, sse2, 1, 0, 1
13241552 AVX_INSTR cmpeqps, sse, 1, 0, 1
13251553 AVX_INSTR cmpeqsd, sse2, 1, 0, 0
@@ -1356,38 +1584,41 @@ AVX_INSTR cmpunordpd, sse2, 1, 0, 1 · 30 unmodified lines
13561584 AVX_INSTR cmpunordps, sse, 1, 0, 1
13571585 AVX_INSTR cmpunordsd, sse2, 1, 0, 0
13581586 AVX_INSTR cmpunordss, sse, 1, 0, 0
1359−AVX_INSTR comisd, sse2
1360−AVX_INSTR comiss, sse
1361−AVX_INSTR cvtdq2pd, sse2
1362−AVX_INSTR cvtdq2ps, sse2
1363−AVX_INSTR cvtpd2dq, sse2
1364−AVX_INSTR cvtpd2ps, sse2
1365−AVX_INSTR cvtps2dq, sse2
1366−AVX_INSTR cvtps2pd, sse2
1367−AVX_INSTR cvtsd2si, sse2
1587+AVX_INSTR comisd, sse2, 1
1588+AVX_INSTR comiss, sse, 1
1589+AVX_INSTR cvtdq2pd, sse2, 1
1590+AVX_INSTR cvtdq2ps, sse2, 1
1591+AVX_INSTR cvtpd2dq, sse2, 1
1592+AVX_INSTR cvtpd2ps, sse2, 1
1593+AVX_INSTR cvtps2dq, sse2, 1
1594+AVX_INSTR cvtps2pd, sse2, 1
1595+AVX_INSTR cvtsd2si, sse2, 1
13681596 AVX_INSTR cvtsd2ss, sse2, 1, 0, 0
13691597 AVX_INSTR cvtsi2sd, sse2, 1, 0, 0
13701598 AVX_INSTR cvtsi2ss, sse, 1, 0, 0
13711599 AVX_INSTR cvtss2sd, sse2, 1, 0, 0
1372−AVX_INSTR cvtss2si, sse
1373−AVX_INSTR cvttpd2dq, sse2
1374−AVX_INSTR cvttps2dq, sse2
1375−AVX_INSTR cvttsd2si, sse2
1376−AVX_INSTR cvttss2si, sse
1600+AVX_INSTR cvtss2si, sse, 1
1601+AVX_INSTR cvttpd2dq, sse2, 1
1602+AVX_INSTR cvttps2dq, sse2, 1
1603+AVX_INSTR cvttsd2si, sse2, 1
1604+AVX_INSTR cvttss2si, sse, 1
13771605 AVX_INSTR divpd, sse2, 1, 0, 0
13781606 AVX_INSTR divps, sse, 1, 0, 0
13791607 AVX_INSTR divsd, sse2, 1, 0, 0
13801608 AVX_INSTR divss, sse, 1, 0, 0
13811609 AVX_INSTR dppd, sse4, 1, 1, 0
13821610 AVX_INSTR dpps, sse4, 1, 1, 0
1383−AVX_INSTR extractps, sse4
1611+AVX_INSTR extractps, sse4, 1
1612+AVX_INSTR gf2p8affineinvqb, gfni, 0, 1, 0
1613+AVX_INSTR gf2p8affineqb, gfni, 0, 1, 0
1614+AVX_INSTR gf2p8mulb, gfni, 0, 0, 0
13841615 AVX_INSTR haddpd, sse3, 1, 0, 0
13851616 AVX_INSTR haddps, sse3, 1, 0, 0
13861617 AVX_INSTR hsubpd, sse3, 1, 0, 0
13871618 AVX_INSTR hsubps, sse3, 1, 0, 0
13881619 AVX_INSTR insertps, sse4, 1, 1, 0
13891620 AVX_INSTR lddqu, sse3
1390−AVX_INSTR ldmxcsr, sse
1621+AVX_INSTR ldmxcsr, sse, 1
13911622 AVX_INSTR maskmovdqu, sse2
13921623 AVX_INSTR maxpd, sse2, 1, 0, 1
13931624 AVX_INSTR maxps, sse, 1, 0, 1
@@ -1397,10 +1628,10 @@ AVX_INSTR minpd, sse2, 1, 0, 1 · 3 unmodified lines
13971628 AVX_INSTR minps, sse, 1, 0, 1
13981629 AVX_INSTR minsd, sse2, 1, 0, 0
13991630 AVX_INSTR minss, sse, 1, 0, 0
1400−AVX_INSTR movapd, sse2
1401−AVX_INSTR movaps, sse
1631+AVX_INSTR movapd, sse2, 1
1632+AVX_INSTR movaps, sse, 1
14021633 AVX_INSTR movd, mmx
1403−AVX_INSTR movddup, sse3
1634+AVX_INSTR movddup, sse3, 1
14041635 AVX_INSTR movdqa, sse2
14051636 AVX_INSTR movdqu, sse2
14061637 AVX_INSTR movhlps, sse, 1, 0, 0
@@ -1409,19 +1640,19 @@ AVX_INSTR movhps, sse, 1, 0, 0 · 2 unmodified lines
14091640 AVX_INSTR movlhps, sse, 1, 0, 0
14101641 AVX_INSTR movlpd, sse2, 1, 0, 0
14111642 AVX_INSTR movlps, sse, 1, 0, 0
1412−AVX_INSTR movmskpd, sse2
1413−AVX_INSTR movmskps, sse
1643+AVX_INSTR movmskpd, sse2, 1
1644+AVX_INSTR movmskps, sse, 1
14141645 AVX_INSTR movntdq, sse2
14151646 AVX_INSTR movntdqa, sse4
1416−AVX_INSTR movntpd, sse2
1417−AVX_INSTR movntps, sse
1647+AVX_INSTR movntpd, sse2, 1
1648+AVX_INSTR movntps, sse, 1
14181649 AVX_INSTR movq, mmx
14191650 AVX_INSTR movsd, sse2, 1, 0, 0
1420−AVX_INSTR movshdup, sse3
1421−AVX_INSTR movsldup, sse3
1651+AVX_INSTR movshdup, sse3, 1
1652+AVX_INSTR movsldup, sse3, 1
14221653 AVX_INSTR movss, sse, 1, 0, 0
1423−AVX_INSTR movupd, sse2
1424−AVX_INSTR movups, sse
1654+AVX_INSTR movupd, sse2, 1
1655+AVX_INSTR movups, sse, 1
14251656 AVX_INSTR mpsadbw, sse4, 0, 1, 0
14261657 AVX_INSTR mulpd, sse2, 1, 0, 1
14271658 AVX_INSTR mulps, sse, 1, 0, 1
@@ -1432,90 +1663,90 @@ AVX_INSTR orps, sse, 1, 0, 1 · 4 unmodified lines
14321663 AVX_INSTR pabsb, ssse3
14331664 AVX_INSTR pabsd, ssse3
14341665 AVX_INSTR pabsw, ssse3
1435−AVX_INSTR packsswb, mmx, 0, 0, 0
14361666 AVX_INSTR packssdw, mmx, 0, 0, 0
1437−AVX_INSTR packuswb, mmx, 0, 0, 0
1667+AVX_INSTR packsswb, mmx, 0, 0, 0
14381668 AVX_INSTR packusdw, sse4, 0, 0, 0
1669+AVX_INSTR packuswb, mmx, 0, 0, 0
14391670 AVX_INSTR paddb, mmx, 0, 0, 1
1440−AVX_INSTR paddw, mmx, 0, 0, 1
14411671 AVX_INSTR paddd, mmx, 0, 0, 1
14421672 AVX_INSTR paddq, sse2, 0, 0, 1
14431673 AVX_INSTR paddsb, mmx, 0, 0, 1
14441674 AVX_INSTR paddsw, mmx, 0, 0, 1
14451675 AVX_INSTR paddusb, mmx, 0, 0, 1
14461676 AVX_INSTR paddusw, mmx, 0, 0, 1
1677+AVX_INSTR paddw, mmx, 0, 0, 1
14471678 AVX_INSTR palignr, ssse3, 0, 1, 0
14481679 AVX_INSTR pand, mmx, 0, 0, 1
14491680 AVX_INSTR pandn, mmx, 0, 0, 0
14501681 AVX_INSTR pavgb, mmx2, 0, 0, 1
14511682 AVX_INSTR pavgw, mmx2, 0, 0, 1
1452−AVX_INSTR pblendvb, sse4 ; can't be emulated
1683+AVX_INSTR pblendvb, sse4, 0, 1, 0 ; last operand must be xmm0 with legacy encoding
14531684 AVX_INSTR pblendw, sse4, 0, 1, 0
1454−AVX_INSTR pclmulqdq, fnord, 0, 1, 0
1455−AVX_INSTR pclmulhqhqdq, fnord, 0, 0, 0
1456−AVX_INSTR pclmulhqlqdq, fnord, 0, 0, 0
1457−AVX_INSTR pclmullqhqdq, fnord, 0, 0, 0
1458−AVX_INSTR pclmullqlqdq, fnord, 0, 0, 0
1459−AVX_INSTR pcmpestri, sse42
1460−AVX_INSTR pcmpestrm, sse42
1461−AVX_INSTR pcmpistri, sse42
1462−AVX_INSTR pcmpistrm, sse42
1685+AVX_INSTR pclmulhqhqdq, clmul, 0, 0, 0
1686+AVX_INSTR pclmulhqlqdq, clmul, 0, 0, 0
1687+AVX_INSTR pclmullqhqdq, clmul, 0, 0, 0
1688+AVX_INSTR pclmullqlqdq, clmul, 0, 0, 0
1689+AVX_INSTR pclmulqdq, clmul, 0, 1, 0
14631690 AVX_INSTR pcmpeqb, mmx, 0, 0, 1
1464−AVX_INSTR pcmpeqw, mmx, 0, 0, 1
14651691 AVX_INSTR pcmpeqd, mmx, 0, 0, 1
14661692 AVX_INSTR pcmpeqq, sse4, 0, 0, 1
1693+AVX_INSTR pcmpeqw, mmx, 0, 0, 1
1694+AVX_INSTR pcmpestri, sse42
1695+AVX_INSTR pcmpestrm, sse42
14671696 AVX_INSTR pcmpgtb, mmx, 0, 0, 0
1468−AVX_INSTR pcmpgtw, mmx, 0, 0, 0
14691697 AVX_INSTR pcmpgtd, mmx, 0, 0, 0
14701698 AVX_INSTR pcmpgtq, sse42, 0, 0, 0
1699+AVX_INSTR pcmpgtw, mmx, 0, 0, 0
1700+AVX_INSTR pcmpistri, sse42
1701+AVX_INSTR pcmpistrm, sse42
14711702 AVX_INSTR pextrb, sse4
14721703 AVX_INSTR pextrd, sse4
14731704 AVX_INSTR pextrq, sse4
14741705 AVX_INSTR pextrw, mmx2
1475−AVX_INSTR phaddw, ssse3, 0, 0, 0
14761706 AVX_INSTR phaddd, ssse3, 0, 0, 0
14771707 AVX_INSTR phaddsw, ssse3, 0, 0, 0
1708+AVX_INSTR phaddw, ssse3, 0, 0, 0
14781709 AVX_INSTR phminposuw, sse4
1479−AVX_INSTR phsubw, ssse3, 0, 0, 0
14801710 AVX_INSTR phsubd, ssse3, 0, 0, 0
14811711 AVX_INSTR phsubsw, ssse3, 0, 0, 0
1712+AVX_INSTR phsubw, ssse3, 0, 0, 0
14821713 AVX_INSTR pinsrb, sse4, 0, 1, 0
14831714 AVX_INSTR pinsrd, sse4, 0, 1, 0
14841715 AVX_INSTR pinsrq, sse4, 0, 1, 0
14851716 AVX_INSTR pinsrw, mmx2, 0, 1, 0
1486−AVX_INSTR pmaddwd, mmx, 0, 0, 1
14871717 AVX_INSTR pmaddubsw, ssse3, 0, 0, 0
1718+AVX_INSTR pmaddwd, mmx, 0, 0, 1
14881719 AVX_INSTR pmaxsb, sse4, 0, 0, 1
1489−AVX_INSTR pmaxsw, mmx2, 0, 0, 1
14901720 AVX_INSTR pmaxsd, sse4, 0, 0, 1
1721+AVX_INSTR pmaxsw, mmx2, 0, 0, 1
14911722 AVX_INSTR pmaxub, mmx2, 0, 0, 1
1492−AVX_INSTR pmaxuw, sse4, 0, 0, 1
14931723 AVX_INSTR pmaxud, sse4, 0, 0, 1
1724+AVX_INSTR pmaxuw, sse4, 0, 0, 1
14941725 AVX_INSTR pminsb, sse4, 0, 0, 1
1495−AVX_INSTR pminsw, mmx2, 0, 0, 1
14961726 AVX_INSTR pminsd, sse4, 0, 0, 1
1727+AVX_INSTR pminsw, mmx2, 0, 0, 1
14971728 AVX_INSTR pminub, mmx2, 0, 0, 1
1498−AVX_INSTR pminuw, sse4, 0, 0, 1
14991729 AVX_INSTR pminud, sse4, 0, 0, 1
1730+AVX_INSTR pminuw, sse4, 0, 0, 1
15001731 AVX_INSTR pmovmskb, mmx2
1501−AVX_INSTR pmovsxbw, sse4
15021732 AVX_INSTR pmovsxbd, sse4
15031733 AVX_INSTR pmovsxbq, sse4
1734+AVX_INSTR pmovsxbw, sse4
1735+AVX_INSTR pmovsxdq, sse4
15041736 AVX_INSTR pmovsxwd, sse4
15051737 AVX_INSTR pmovsxwq, sse4
1506−AVX_INSTR pmovsxdq, sse4
1507−AVX_INSTR pmovzxbw, sse4
15081738 AVX_INSTR pmovzxbd, sse4
15091739 AVX_INSTR pmovzxbq, sse4
1740+AVX_INSTR pmovzxbw, sse4
1741+AVX_INSTR pmovzxdq, sse4
15101742 AVX_INSTR pmovzxwd, sse4
15111743 AVX_INSTR pmovzxwq, sse4
1512−AVX_INSTR pmovzxdq, sse4
15131744 AVX_INSTR pmuldq, sse4, 0, 0, 1
15141745 AVX_INSTR pmulhrsw, ssse3, 0, 0, 1
15151746 AVX_INSTR pmulhuw, mmx2, 0, 0, 1
15161747 AVX_INSTR pmulhw, mmx, 0, 0, 1
1517−AVX_INSTR pmullw, mmx, 0, 0, 1
15181748 AVX_INSTR pmulld, sse4, 0, 0, 1
1749+AVX_INSTR pmullw, mmx, 0, 0, 1
15191750 AVX_INSTR pmuludq, sse2, 0, 0, 1
15201751 AVX_INSTR por, mmx, 0, 0, 1
15211752 AVX_INSTR psadbw, mmx2, 0, 0, 1
@@ -1524,57 +1755,57 @@ AVX_INSTR pshufd, sse2 · 2 unmodified lines
15241755 AVX_INSTR pshufhw, sse2
15251756 AVX_INSTR pshuflw, sse2
15261757 AVX_INSTR psignb, ssse3, 0, 0, 0
1527−AVX_INSTR psignw, ssse3, 0, 0, 0
15281758 AVX_INSTR psignd, ssse3, 0, 0, 0
1529−AVX_INSTR psllw, mmx, 0, 0, 0
1759+AVX_INSTR psignw, ssse3, 0, 0, 0
15301760 AVX_INSTR pslld, mmx, 0, 0, 0
1531−AVX_INSTR psllq, mmx, 0, 0, 0
15321761 AVX_INSTR pslldq, sse2, 0, 0, 0
1533−AVX_INSTR psraw, mmx, 0, 0, 0
1762+AVX_INSTR psllq, mmx, 0, 0, 0
1763+AVX_INSTR psllw, mmx, 0, 0, 0
15341764 AVX_INSTR psrad, mmx, 0, 0, 0
1535−AVX_INSTR psrlw, mmx, 0, 0, 0
1765+AVX_INSTR psraw, mmx, 0, 0, 0
15361766 AVX_INSTR psrld, mmx, 0, 0, 0
1537−AVX_INSTR psrlq, mmx, 0, 0, 0
15381767 AVX_INSTR psrldq, sse2, 0, 0, 0
1768+AVX_INSTR psrlq, mmx, 0, 0, 0
1769+AVX_INSTR psrlw, mmx, 0, 0, 0
15391770 AVX_INSTR psubb, mmx, 0, 0, 0
1540−AVX_INSTR psubw, mmx, 0, 0, 0
15411771 AVX_INSTR psubd, mmx, 0, 0, 0
15421772 AVX_INSTR psubq, sse2, 0, 0, 0
15431773 AVX_INSTR psubsb, mmx, 0, 0, 0
15441774 AVX_INSTR psubsw, mmx, 0, 0, 0
15451775 AVX_INSTR psubusb, mmx, 0, 0, 0
15461776 AVX_INSTR psubusw, mmx, 0, 0, 0
1777+AVX_INSTR psubw, mmx, 0, 0, 0
15471778 AVX_INSTR ptest, sse4
15481779 AVX_INSTR punpckhbw, mmx, 0, 0, 0
1549−AVX_INSTR punpckhwd, mmx, 0, 0, 0
15501780 AVX_INSTR punpckhdq, mmx, 0, 0, 0
15511781 AVX_INSTR punpckhqdq, sse2, 0, 0, 0
1782+AVX_INSTR punpckhwd, mmx, 0, 0, 0
15521783 AVX_INSTR punpcklbw, mmx, 0, 0, 0
1553−AVX_INSTR punpcklwd, mmx, 0, 0, 0
15541784 AVX_INSTR punpckldq, mmx, 0, 0, 0
15551785 AVX_INSTR punpcklqdq, sse2, 0, 0, 0
1786+AVX_INSTR punpcklwd, mmx, 0, 0, 0
15561787 AVX_INSTR pxor, mmx, 0, 0, 1
1557−AVX_INSTR rcpps, sse
1788+AVX_INSTR rcpps, sse, 1
15581789 AVX_INSTR rcpss, sse, 1, 0, 0
1559−AVX_INSTR roundpd, sse4
1560−AVX_INSTR roundps, sse4
1790+AVX_INSTR roundpd, sse4, 1
1791+AVX_INSTR roundps, sse4, 1
15611792 AVX_INSTR roundsd, sse4, 1, 1, 0
15621793 AVX_INSTR roundss, sse4, 1, 1, 0
1563−AVX_INSTR rsqrtps, sse
1794+AVX_INSTR rsqrtps, sse, 1
15641795 AVX_INSTR rsqrtss, sse, 1, 0, 0
15651796 AVX_INSTR shufpd, sse2, 1, 1, 0
15661797 AVX_INSTR shufps, sse, 1, 1, 0
1567−AVX_INSTR sqrtpd, sse2
1568−AVX_INSTR sqrtps, sse
1798+AVX_INSTR sqrtpd, sse2, 1
1799+AVX_INSTR sqrtps, sse, 1
15691800 AVX_INSTR sqrtsd, sse2, 1, 0, 0
15701801 AVX_INSTR sqrtss, sse, 1, 0, 0
1571−AVX_INSTR stmxcsr, sse
1802+AVX_INSTR stmxcsr, sse, 1
15721803 AVX_INSTR subpd, sse2, 1, 0, 0
15731804 AVX_INSTR subps, sse, 1, 0, 0
15741805 AVX_INSTR subsd, sse2, 1, 0, 0
15751806 AVX_INSTR subss, sse, 1, 0, 0
1576−AVX_INSTR ucomisd, sse2
1577−AVX_INSTR ucomiss, sse
1807+AVX_INSTR ucomisd, sse2, 1
1808+AVX_INSTR ucomiss, sse, 1
15781809 AVX_INSTR unpckhpd, sse2, 1, 0, 0
15791810 AVX_INSTR unpckhps, sse, 1, 0, 0
15801811 AVX_INSTR unpcklpd, sse2, 1, 0, 0
@@ -1584,8 +1815,41 @@ AVX_INSTR xorps, sse, 1, 0, 1 · 3 unmodified lines
15841815
15851816 ; 3DNow instructions, for sharing code between AVX, SSE and 3DN
15861817 AVX_INSTR pfadd, 3dnow, 1, 0, 1
1587−AVX_INSTR pfsub, 3dnow, 1, 0, 0
15881818 AVX_INSTR pfmul, 3dnow, 1, 0, 1
1819+AVX_INSTR pfsub, 3dnow, 1, 0, 0
1820+
1821+;%1 == instruction
1822+;%2 == minimal instruction set
1823+%macro GPR_INSTR 2
1824+ %macro %1 2-5 fnord, %1, %2
1825+ %ifdef cpuname
1826+ %if notcpuflag(%5)
1827+ %error use of ``%4'' %5 instruction in cpuname function: current_function
1828+ %endif
1829+ %endif
1830+ %ifidn %3, fnord
1831+ %4 %1, %2
1832+ %else
1833+ %4 %1, %2, %3
1834+ %endif
1835+ %endmacro
1836+%endmacro
1837+
1838+GPR_INSTR andn, bmi1
1839+GPR_INSTR bextr, bmi1
1840+GPR_INSTR blsi, bmi1
1841+GPR_INSTR blsmsk, bmi1
1842+GPR_INSTR blsr, bmi1
1843+GPR_INSTR bzhi, bmi2
1844+GPR_INSTR crc32, sse42
1845+GPR_INSTR mulx, bmi2
1846+GPR_INSTR pdep, bmi2
1847+GPR_INSTR pext, bmi2
1848+GPR_INSTR popcnt, sse42
1849+GPR_INSTR rorx, bmi2
1850+GPR_INSTR sarx, bmi2
1851+GPR_INSTR shlx, bmi2
1852+GPR_INSTR shrx, bmi2
15891853
15901854 ; base-4 constants for shuffles
15911855 %assign i 0
@@ -1618,15 +1882,11 @@ AVX_INSTR pfmul, 3dnow, 1, 0, 1 · 26 unmodified lines
16181882 %endmacro
16191883 %endmacro
16201884
1621−FMA_INSTR pmacsww, pmullw, paddw
1622−FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation
1623−FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation
1885+FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation
1886+FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation
1887+FMA_INSTR pmacsww, pmullw, paddw
16241888 FMA_INSTR pmadcswd, pmaddwd, paddd
16251889
1626−; tzcnt is equivalent to "rep bsf" and is backwards-compatible with bsf.
1627−; This lets us use tzcnt without bumping the yasm version requirement yet.
1628−%define tzcnt rep bsf
1629−
16301890 ; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax.
16311891 ; FMA3 is only possible if dst is the same as one of the src registers.
16321892 ; Either src2 or src3 can be a memory operand.
@@ -1687,6 +1947,11 @@ FMA4_INSTR fnmsub, pd, ps, sd, ss · 54 unmodified lines
16871947 %assign %%evex_required 1
16881948 %endif
16891949 %endif
1950+ %ifnum regnumof%3
1951+ %if regnumof%3 >= 16 || sizeof%3 > 32
1952+ %assign %%evex_required 1
1953+ %endif
1954+ %endif
16901955 %if %%evex_required
16911956 %6 %%args
16921957 %else
@@ -1711,16 +1976,3 @@ EVEX_INSTR vrcpps, vrcp14ps, 1 ; EVEX versions have higher precision · 18 unmodified lines
17111976 EVEX_INSTR vrcpss, vrcp14ss, 1
17121977 EVEX_INSTR vrsqrtps, vrsqrt14ps, 1
17131978 EVEX_INSTR vrsqrtss, vrsqrt14ss, 1
1714−
1715−; workaround: vpbroadcastq is broken in x86_32 due to a yasm bug (fixed in 1.3.0)
1716−%ifdef __YASM_VER__
1717− %if __YASM_VERSION_ID__ < 0x01030000 && ARCH_X86_64 == 0
1718− %macro vpbroadcastq 2
1719− %if sizeof%1 == 16
1720− movddup %1, %2
1721− %else
1722− vbroadcastsd %1, %2
1723− %endif
1724− %endmacro
1725− %endif
1726−%endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t