| @@ -1,7 +1,7 @@ |
| 1 | 1 | ;***************************************************************************** |
| 2 | | −;* x86inc.asm: x264asm abstraction layer |
| 2 | +;* x86inc.asm: x86 abstraction layer |
| 3 | 3 | ;***************************************************************************** |
| 4 | | −;* Copyright (C) 2005-2018 x264 project |
| 4 | +;* Copyright (C) 2005-2024 x264 project |
| 5 | 5 | ;* |
| 6 | 6 | ;* Authors: Loren Merritt <lorenm@u.washington.edu> |
| 7 | 7 | ;* Henrik Gramner <henrik@gramner.com> |
| @@ -21,21 +21,14 @@ · 13 unmodified lines |
| 21 | 21 | ;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 22 | 22 | ;***************************************************************************** |
| 23 | 23 | |
| 24 | | −; This is a header file for the x264ASM assembly language, which uses |
| 24 | +; This is a header file for the x86inc.asm assembly language, which uses |
| 25 | 25 | ; NASM/YASM syntax combined with a large number of macros to provide easy |
| 26 | 26 | ; abstraction between different calling conventions (x86_32, win64, linux64). |
| 27 | 27 | ; It also has various other useful features to simplify writing the kind of |
| 28 | | −; DSP functions that are most often used in x264. |
| 29 | | − |
| 30 | | −; Unlike the rest of x264, this file is available under an ISC license, as it |
| 31 | | −; has significant usefulness outside of x264 and we want it to be available |
| 32 | | −; to the largest audience possible. Of course, if you modify it for your own |
| 33 | | −; purposes to add a new feature, we strongly encourage contributing a patch |
| 34 | | −; as this feature might be useful for others as well. Send patches or ideas |
| 35 | | −; to x264-devel@videolan.org . |
| 28 | +; DSP functions that are most often used. |
| 36 | 29 | |
| 37 | 30 | %ifndef private_prefix |
| 38 | | − %define private_prefix x264 |
| 31 | + %error private_prefix not defined |
| 39 | 32 | %endif |
| 40 | 33 | |
| 41 | 34 | %ifndef public_prefix |
| @@ -68,12 +61,19 @@ · 26 unmodified lines |
| 68 | 61 | %endif |
| 69 | 62 | |
| 70 | 63 | %define FORMAT_ELF 0 |
| 64 | +%define FORMAT_MACHO 0 |
| 71 | 65 | %ifidn __OUTPUT_FORMAT__,elf |
| 72 | 66 | %define FORMAT_ELF 1 |
| 73 | 67 | %elifidn __OUTPUT_FORMAT__,elf32 |
| 74 | 68 | %define FORMAT_ELF 1 |
| 75 | 69 | %elifidn __OUTPUT_FORMAT__,elf64 |
| 76 | 70 | %define FORMAT_ELF 1 |
| 71 | +%elifidn __OUTPUT_FORMAT__,macho |
| 72 | + %define FORMAT_MACHO 1 |
| 73 | +%elifidn __OUTPUT_FORMAT__,macho32 |
| 74 | + %define FORMAT_MACHO 1 |
| 75 | +%elifidn __OUTPUT_FORMAT__,macho64 |
| 76 | + %define FORMAT_MACHO 1 |
| 77 | 77 | %endif |
| 78 | 78 | |
| 79 | 79 | %ifdef PREFIX |
| @@ -82,6 +82,11 @@ · 2 unmodified lines |
| 82 | 82 | %define mangle(x) x |
| 83 | 83 | %endif |
| 84 | 84 | |
| 85 | +; Use VEX-encoding even in non-AVX functions |
| 86 | +%ifndef FORCE_VEX_ENCODING |
| 87 | + %define FORCE_VEX_ENCODING 0 |
| 88 | +%endif |
| 89 | + |
| 85 | 90 | ; aout does not support align= |
| 86 | 91 | ; NOTE: This section is out of sync with x264, in order to |
| 87 | 92 | ; keep supporting OS/2. |
| @@ -99,28 +104,27 @@ · 11 unmodified lines |
| 99 | 104 | %endif |
| 100 | 105 | %endmacro |
| 101 | 106 | |
| 102 | | −%if WIN64 |
| 103 | | − %define PIC |
| 104 | | −%elif ARCH_X86_64 == 0 |
| 105 | | −; x86_32 doesn't require PIC. |
| 106 | | −; Some distros prefer shared objects to be PIC, but nothing breaks if |
| 107 | | −; the code contains a few textrels, so we'll skip that complexity. |
| 108 | | − %undef PIC |
| 109 | | −%endif |
| 110 | | −%ifdef PIC |
| 107 | +%if ARCH_X86_64 |
| 108 | + %define PIC 1 ; always use PIC on x86-64 |
| 111 | 109 | default rel |
| 110 | +%elifidn __OUTPUT_FORMAT__,win32 |
| 111 | + %define PIC 0 ; PIC isn't used on 32-bit Windows |
| 112 | +%elifndef PIC |
| 113 | + %define PIC 0 |
| 112 | 114 | %endif |
| 113 | 115 | |
| 114 | | −%macro CPUNOP 1 |
| 115 | | − %if HAVE_CPUNOP |
| 116 | | − CPU %1 |
| 116 | +%define HAVE_PRIVATE_EXTERN 1 |
| 117 | +%ifdef __NASM_VERSION_ID__ |
| 118 | + %use smartalign |
| 119 | + %if __NASM_VERSION_ID__ < 0x020e0000 ; 2.14 |
| 120 | + %define HAVE_PRIVATE_EXTERN 0 |
| 117 | 121 | %endif |
| 118 | | −%endmacro |
| 122 | +%endif |
| 119 | 123 | |
| 120 | 124 | ; Macros to eliminate most code duplication between x86_32 and x86_64: |
| 121 | 125 | ; Currently this works only for leaf functions which load all their arguments |
| 122 | 126 | ; into registers at the start, and make no other use of the stack. Luckily that |
| 123 | | −; covers most of x264's asm. |
| 127 | +; covers most use cases. |
| 124 | 128 | |
| 125 | 129 | ; PROLOGUE: |
| 126 | 130 | ; %1 = number of arguments. loads them from stack if needed. |
| @@ -232,6 +236,18 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 105 unmodified lines |
| 232 | 236 | %define gprsize 4 |
| 233 | 237 | %endif |
| 234 | 238 | |
| 239 | +%macro LEA 2 |
| 240 | +%if ARCH_X86_64 |
| 241 | + lea %1, [%2] |
| 242 | +%elif PIC |
| 243 | + call $+5 ; special-cased to not affect the RSB on most CPU:s |
| 244 | + pop %1 |
| 245 | + add %1, (%2)-$+1 |
| 246 | +%else |
| 247 | + mov %1, %2 |
| 248 | +%endif |
| 249 | +%endmacro |
| 250 | + |
| 235 | 251 | ; Repeats an instruction/operation for multiple arguments. |
| 236 | 252 | ; Example usage: "REPX {psrlw x, 8}, m0, m1, m2, m3" |
| 237 | 253 | %macro REPX 2-* ; operation, args |
| @@ -303,6 +319,10 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 65 unmodified lines |
| 303 | 319 | %endif |
| 304 | 320 | %endmacro |
| 305 | 321 | |
| 322 | +%if ARCH_X86_64 == 0 |
| 323 | + %define movsxd movifnidn |
| 324 | +%endif |
| 325 | + |
| 306 | 326 | %macro movsxdifnidn 2 |
| 307 | 327 | %ifnidn %1, %2 |
| 308 | 328 | movsxd %1, %2 |
| @@ -354,7 +374,46 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 45 unmodified lines |
| 354 | 374 | %define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512))) |
| 355 | 375 | %define high_mm_regs (16*cpuflag(avx512)) |
| 356 | 376 | |
| 357 | | −%macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only) |
| 377 | +; Large stack allocations on Windows need to use stack probing in order |
| 378 | +; to guarantee that all stack memory is committed before accessing it. |
| 379 | +; This is done by ensuring that the guard page(s) at the end of the |
| 380 | +; currently committed pages are touched prior to any pages beyond that. |
| 381 | +%if WIN64 |
| 382 | + %assign STACK_PROBE_SIZE 8192 |
| 383 | +%elifidn __OUTPUT_FORMAT__, win32 |
| 384 | + %assign STACK_PROBE_SIZE 4096 |
| 385 | +%else |
| 386 | + %assign STACK_PROBE_SIZE 0 |
| 387 | +%endif |
| 388 | + |
| 389 | +%macro PROBE_STACK 1 ; stack_size |
| 390 | + %if STACK_PROBE_SIZE |
| 391 | + %assign %%i STACK_PROBE_SIZE |
| 392 | + %rep %1 / STACK_PROBE_SIZE |
| 393 | + mov eax, [rsp-%%i] |
| 394 | + %assign %%i %%i+STACK_PROBE_SIZE |
| 395 | + %endrep |
| 396 | + %endif |
| 397 | +%endmacro |
| 398 | + |
| 399 | +%macro RESET_STACK_STATE 0 |
| 400 | + %ifidn rstk, rsp |
| 401 | + %assign stack_offset stack_offset - stack_size_padded |
| 402 | + %else |
| 403 | + %xdefine rstk rsp |
| 404 | + %endif |
| 405 | + %assign stack_size 0 |
| 406 | + %assign stack_size_padded 0 |
| 407 | + %assign xmm_regs_used 0 |
| 408 | +%endmacro |
| 409 | + |
| 410 | +%macro ALLOC_STACK 0-2 0, 0 ; stack_size, n_xmm_regs |
| 411 | + RESET_STACK_STATE |
| 412 | + %ifnum %2 |
| 413 | + %if mmsize != 8 |
| 414 | + %assign xmm_regs_used %2 |
| 415 | + %endif |
| 416 | + %endif |
| 358 | 417 | %ifnum %1 |
| 359 | 418 | %if %1 != 0 |
| 360 | 419 | %assign %%pad 0 |
| @@ -364,16 +423,14 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 3 unmodified lines |
| 364 | 423 | %endif |
| 365 | 424 | %if WIN64 |
| 366 | 425 | %assign %%pad %%pad + 32 ; shadow space |
| 367 | | − %if mmsize != 8 |
| 368 | | − %assign xmm_regs_used %2 |
| 369 | | − %if xmm_regs_used > 8 |
| 370 | | − %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers |
| 371 | | − %endif |
| 426 | + %if xmm_regs_used > 8 |
| 427 | + %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers |
| 372 | 428 | %endif |
| 373 | 429 | %endif |
| 374 | 430 | %if required_stack_alignment <= STACK_ALIGNMENT |
| 375 | 431 | ; maintain the current stack alignment |
| 376 | 432 | %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) |
| 433 | + PROBE_STACK stack_size_padded |
| 377 | 434 | SUB rsp, stack_size_padded |
| 378 | 435 | %else |
| 379 | 436 | %assign %%reg_num (regs_used - 1) |
| @@ -389,6 +446,7 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 9 unmodified lines |
| 389 | 446 | %xdefine rstkm rstk |
| 390 | 447 | %endif |
| 391 | 448 | %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1)) |
| 449 | + PROBE_STACK stack_size_padded |
| 392 | 450 | mov rstk, rsp |
| 393 | 451 | and rsp, ~(required_stack_alignment-1) |
| 394 | 452 | sub rsp, stack_size_padded |
| @@ -399,7 +457,7 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 4 unmodified lines |
| 399 | 457 | %endif |
| 400 | 458 | %endmacro |
| 401 | 459 | |
| 402 | | −%macro SETUP_STACK_POINTER 1 |
| 460 | +%macro SETUP_STACK_POINTER 0-1 0 |
| 403 | 461 | %ifnum %1 |
| 404 | 462 | %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT |
| 405 | 463 | %if %1 > 0 |
| @@ -462,35 +520,62 @@ DECLARE_REG 14, R13, 120 · 56 unmodified lines |
| 462 | 520 | %endif |
| 463 | 521 | %endmacro |
| 464 | 522 | |
| 465 | | −%macro WIN64_PUSH_XMM 0 |
| 466 | | − ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. |
| 467 | | − %if xmm_regs_used > 6 + high_mm_regs |
| 468 | | − movaps [rstk + stack_offset + 8], xmm6 |
| 469 | | − %endif |
| 470 | | − %if xmm_regs_used > 7 + high_mm_regs |
| 471 | | − movaps [rstk + stack_offset + 24], xmm7 |
| 472 | | − %endif |
| 473 | | − %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8 |
| 474 | | − %if %%xmm_regs_on_stack > 0 |
| 475 | | − %assign %%i 8 |
| 476 | | − %rep %%xmm_regs_on_stack |
| 477 | | − movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i |
| 478 | | − %assign %%i %%i+1 |
| 479 | | − %endrep |
| 523 | +; Push XMM registers to the stack. If no argument is specified all used register |
| 524 | +; will be pushed, otherwise only push previously unpushed registers. |
| 525 | +%macro WIN64_PUSH_XMM 0-2 ; new_xmm_regs_used, xmm_regs_pushed |
| 526 | + %if mmsize != 8 |
| 527 | + %if %0 == 2 |
| 528 | + %assign %%pushed %2 |
| 529 | + %assign xmm_regs_used %1 |
| 530 | + %elif %0 == 1 |
| 531 | + %assign %%pushed xmm_regs_used |
| 532 | + %assign xmm_regs_used %1 |
| 533 | + %else |
| 534 | + %assign %%pushed 0 |
| 535 | + %endif |
| 536 | + ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. |
| 537 | + %if %%pushed <= 6 + high_mm_regs && xmm_regs_used > 6 + high_mm_regs |
| 538 | + movaps [rstk + stack_offset + 8], xmm6 |
| 539 | + %endif |
| 540 | + %if %%pushed <= 7 + high_mm_regs && xmm_regs_used > 7 + high_mm_regs |
| 541 | + movaps [rstk + stack_offset + 24], xmm7 |
| 542 | + %endif |
| 543 | + %assign %%pushed %%pushed - high_mm_regs - 8 |
| 544 | + %if %%pushed < 0 |
| 545 | + %assign %%pushed 0 |
| 546 | + %endif |
| 547 | + %assign %%regs_to_push xmm_regs_used - %%pushed - high_mm_regs - 8 |
| 548 | + %if %%regs_to_push > 0 |
| 549 | + ASSERT (%%regs_to_push + %%pushed) * 16 <= stack_size_padded - stack_size - 32 |
| 550 | + %assign %%i %%pushed + 8 |
| 551 | + %rep %%regs_to_push |
| 552 | + movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i |
| 553 | + %assign %%i %%i+1 |
| 554 | + %endrep |
| 555 | + %endif |
| 480 | 556 | %endif |
| 481 | 557 | %endmacro |
| 482 | 558 | |
| 483 | | −%macro WIN64_SPILL_XMM 1 |
| 484 | | − %assign xmm_regs_used %1 |
| 485 | | − ASSERT xmm_regs_used <= 16 + high_mm_regs |
| 486 | | − %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8 |
| 487 | | − %if %%xmm_regs_on_stack > 0 |
| 488 | | − ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack. |
| 489 | | − %assign %%pad %%xmm_regs_on_stack*16 + 32 |
| 490 | | − %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) |
| 491 | | − SUB rsp, stack_size_padded |
| 559 | +; Allocated stack space for XMM registers and push all, or a subset, of those |
| 560 | +%macro WIN64_SPILL_XMM 1-2 ; xmm_regs_used, xmm_regs_reserved |
| 561 | + RESET_STACK_STATE |
| 562 | + %if mmsize != 8 |
| 563 | + %assign xmm_regs_used %1 |
| 564 | + ASSERT xmm_regs_used <= 16 + high_mm_regs |
| 565 | + %if %0 == 2 |
| 566 | + ASSERT %2 >= %1 |
| 567 | + %assign %%xmm_regs_on_stack %2 - high_mm_regs - 8 |
| 568 | + %else |
| 569 | + %assign %%xmm_regs_on_stack %1 - high_mm_regs - 8 |
| 570 | + %endif |
| 571 | + %if %%xmm_regs_on_stack > 0 |
| 572 | + ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack. |
| 573 | + %assign %%pad %%xmm_regs_on_stack*16 + 32 |
| 574 | + %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) |
| 575 | + SUB rsp, stack_size_padded |
| 576 | + %endif |
| 577 | + WIN64_PUSH_XMM |
| 492 | 578 | %endif |
| 493 | | − WIN64_PUSH_XMM |
| 494 | 579 | %endmacro |
| 495 | 580 | |
| 496 | 581 | %macro WIN64_RESTORE_XMM_INTERNAL 0 |
| @@ -521,9 +606,7 @@ DECLARE_REG 14, R13, 120 · 24 unmodified lines |
| 521 | 606 | |
| 522 | 607 | %macro WIN64_RESTORE_XMM 0 |
| 523 | 608 | WIN64_RESTORE_XMM_INTERNAL |
| 524 | | − %assign stack_offset (stack_offset-stack_size_padded) |
| 525 | | − %assign stack_size_padded 0 |
| 526 | | − %assign xmm_regs_used 0 |
| 609 | + RESET_STACK_STATE |
| 527 | 610 | %endmacro |
| 528 | 611 | |
| 529 | 612 | %define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs |
| @@ -558,12 +641,11 @@ DECLARE_REG 14, R13, 72 · 28 unmodified lines |
| 558 | 641 | %macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... |
| 559 | 642 | %assign num_args %1 |
| 560 | 643 | %assign regs_used %2 |
| 561 | | − %assign xmm_regs_used %3 |
| 562 | 644 | ASSERT regs_used >= num_args |
| 563 | 645 | SETUP_STACK_POINTER %4 |
| 564 | 646 | ASSERT regs_used <= 15 |
| 565 | 647 | PUSH_IF_USED 9, 10, 11, 12, 13, 14 |
| 566 | | − ALLOC_STACK %4 |
| 648 | + ALLOC_STACK %4, %3 |
| 567 | 649 | LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 |
| 568 | 650 | %if %0 > 4 |
| 569 | 651 | %ifnum %4 |
| @@ -627,7 +709,7 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 57 unmodified lines |
| 627 | 709 | SETUP_STACK_POINTER %4 |
| 628 | 710 | ASSERT regs_used <= 7 |
| 629 | 711 | PUSH_IF_USED 3, 4, 5, 6 |
| 630 | | − ALLOC_STACK %4 |
| 712 | + ALLOC_STACK %4, %3 |
| 631 | 713 | LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 |
| 632 | 714 | %if %0 > 4 |
| 633 | 715 | %ifnum %4 |
| @@ -660,11 +742,19 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 26 unmodified lines |
| 660 | 742 | %endif ;====================================================================== |
| 661 | 743 | |
| 662 | 744 | %if WIN64 == 0 |
| 663 | | − %macro WIN64_SPILL_XMM 1 |
| 745 | + %macro WIN64_SPILL_XMM 1-2 |
| 746 | + RESET_STACK_STATE |
| 747 | + %if mmsize != 8 |
| 748 | + %assign xmm_regs_used %1 |
| 749 | + %endif |
| 664 | 750 | %endmacro |
| 665 | 751 | %macro WIN64_RESTORE_XMM 0 |
| 752 | + RESET_STACK_STATE |
| 666 | 753 | %endmacro |
| 667 | | − %macro WIN64_PUSH_XMM 0 |
| 754 | + %macro WIN64_PUSH_XMM 0-2 |
| 755 | + %if mmsize != 8 && %0 >= 1 |
| 756 | + %assign xmm_regs_used %1 |
| 757 | + %endif |
| 668 | 758 | %endmacro |
| 669 | 759 | %endif |
| 670 | 760 | |
| @@ -705,7 +795,7 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 34 unmodified lines |
| 705 | 795 | |
| 706 | 796 | BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp |
| 707 | 797 | |
| 708 | | −%macro TAIL_CALL 2 ; callee, is_nonadjacent |
| 798 | +%macro TAIL_CALL 1-2 1 ; callee, is_nonadjacent |
| 709 | 799 | %if has_epilogue |
| 710 | 800 | call %1 |
| 711 | 801 | RET |
| @@ -735,22 +825,25 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 23 unmodified lines |
| 735 | 825 | %endmacro |
| 736 | 826 | %macro cglobal_internal 2-3+ |
| 737 | 827 | annotate_function_size |
| 738 | | − %if %1 |
| 739 | | − %xdefine %%FUNCTION_PREFIX private_prefix |
| 740 | | − %xdefine %%VISIBILITY hidden |
| 741 | | − %else |
| 742 | | − %xdefine %%FUNCTION_PREFIX public_prefix |
| 743 | | − %xdefine %%VISIBILITY |
| 744 | | − %endif |
| 745 | 828 | %ifndef cglobaled_%2 |
| 746 | | − %xdefine %2 mangle(%%FUNCTION_PREFIX %+ _ %+ %2) |
| 829 | + %if %1 |
| 830 | + %xdefine %2 mangle(private_prefix %+ _ %+ %2) |
| 831 | + %else |
| 832 | + %xdefine %2 mangle(public_prefix %+ _ %+ %2) |
| 833 | + %endif |
| 747 | 834 | %xdefine %2.skip_prologue %2 %+ .skip_prologue |
| 748 | 835 | CAT_XDEFINE cglobaled_, %2, 1 |
| 749 | 836 | %endif |
| 750 | 837 | %xdefine current_function %2 |
| 751 | 838 | %xdefine current_function_section __SECT__ |
| 752 | 839 | %if FORMAT_ELF |
| 753 | | − global %2:function %%VISIBILITY |
| 840 | + %if %1 |
| 841 | + global %2:function hidden |
| 842 | + %else |
| 843 | + global %2:function |
| 844 | + %endif |
| 845 | + %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN && %1 |
| 846 | + global %2:private_extern |
| 754 | 847 | %else |
| 755 | 848 | global %2 |
| 756 | 849 | %endif |
| @@ -771,6 +864,8 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 14 unmodified lines |
| 771 | 864 | %macro cglobal_label 1 |
| 772 | 865 | %if FORMAT_ELF |
| 773 | 866 | global current_function %+ %1:function hidden |
| 867 | + %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN |
| 868 | + global current_function %+ %1:private_extern |
| 774 | 869 | %else |
| 775 | 870 | global current_function %+ %1 |
| 776 | 871 | %endif |
| @@ -796,15 +891,34 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 19 unmodified lines |
| 796 | 891 | %xdefine %1 mangle(private_prefix %+ _ %+ %1) |
| 797 | 892 | %if FORMAT_ELF |
| 798 | 893 | global %1:data hidden |
| 894 | + %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN |
| 895 | + global %1:private_extern |
| 799 | 896 | %else |
| 800 | 897 | global %1 |
| 801 | 898 | %endif |
| 802 | 899 | %1: %2 |
| 803 | 900 | %endmacro |
| 804 | 901 | |
| 805 | | −; This is needed for ELF, otherwise the GNU linker assumes the stack is executable by default. |
| 806 | 902 | %if FORMAT_ELF |
| 903 | + ; The GNU linker assumes the stack is executable by default. |
| 807 | 904 | [SECTION .note.GNU-stack noalloc noexec nowrite progbits] |
| 905 | + |
| 906 | + %ifdef __NASM_VERSION_ID__ |
| 907 | + %if __NASM_VERSION_ID__ >= 0x020e0300 ; 2.14.03 |
| 908 | + %if ARCH_X86_64 |
| 909 | + ; Control-flow Enforcement Technology (CET) properties. |
| 910 | + [SECTION .note.gnu.property alloc noexec nowrite note align=gprsize] |
| 911 | + dd 0x00000004 ; n_namesz |
| 912 | + dd gprsize + 8 ; n_descsz |
| 913 | + dd 0x00000005 ; n_type = NT_GNU_PROPERTY_TYPE_0 |
| 914 | + db "GNU",0 ; n_name |
| 915 | + dd 0xc0000002 ; pr_type = GNU_PROPERTY_X86_FEATURE_1_AND |
| 916 | + dd 0x00000004 ; pr_datasz |
| 917 | + dd 0x00000002 ; pr_data = GNU_PROPERTY_X86_FEATURE_1_SHSTK |
| 918 | + dd 0x00000000 ; pr_padding |
| 919 | + %endif |
| 920 | + %endif |
| 921 | + %endif |
| 808 | 922 | %endif |
| 809 | 923 | |
| 810 | 924 | ; Tell debuggers how large the function was. |
| @@ -828,32 +942,34 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 17 unmodified lines |
| 828 | 942 | ; cpuflags |
| 829 | 943 | |
| 830 | 944 | %assign cpuflags_mmx (1<<0) |
| 831 | | −%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx |
| 832 | | −%assign cpuflags_3dnow (1<<2) | cpuflags_mmx |
| 833 | | −%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow |
| 834 | | −%assign cpuflags_sse (1<<4) | cpuflags_mmx2 |
| 835 | | −%assign cpuflags_sse2 (1<<5) | cpuflags_sse |
| 836 | | −%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 |
| 837 | | −%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2 |
| 838 | | −%assign cpuflags_sse3 (1<<8) | cpuflags_sse2 |
| 839 | | −%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3 |
| 840 | | −%assign cpuflags_sse4 (1<<10)| cpuflags_ssse3 |
| 841 | | −%assign cpuflags_sse42 (1<<11)| cpuflags_sse4 |
| 842 | | −%assign cpuflags_aesni (1<<12)| cpuflags_sse42 |
| 843 | | −%assign cpuflags_avx (1<<13)| cpuflags_sse42 |
| 844 | | −%assign cpuflags_xop (1<<14)| cpuflags_avx |
| 845 | | −%assign cpuflags_fma4 (1<<15)| cpuflags_avx |
| 846 | | −%assign cpuflags_fma3 (1<<16)| cpuflags_avx |
| 847 | | −%assign cpuflags_bmi1 (1<<17)| cpuflags_avx|cpuflags_lzcnt |
| 848 | | −%assign cpuflags_bmi2 (1<<18)| cpuflags_bmi1 |
| 849 | | −%assign cpuflags_avx2 (1<<19)| cpuflags_fma3|cpuflags_bmi2 |
| 850 | | −%assign cpuflags_avx512 (1<<20)| cpuflags_avx2 ; F, CD, BW, DQ, VL |
| 851 | | −%assign cpuflags_avx512icl (1<<25)| cpuflags_avx512 |
| 852 | | − |
| 853 | | −%assign cpuflags_cache32 (1<<21) |
| 854 | | −%assign cpuflags_cache64 (1<<22) |
| 855 | | −%assign cpuflags_aligned (1<<23) ; not a cpu feature, but a function variant |
| 856 | | −%assign cpuflags_atom (1<<24) |
| 945 | +%assign cpuflags_mmx2 (1<<1) | cpuflags_mmx |
| 946 | +%assign cpuflags_3dnow (1<<2) | cpuflags_mmx |
| 947 | +%assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow |
| 948 | +%assign cpuflags_sse (1<<4) | cpuflags_mmx2 |
| 949 | +%assign cpuflags_sse2 (1<<5) | cpuflags_sse |
| 950 | +%assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 |
| 951 | +%assign cpuflags_lzcnt (1<<7) | cpuflags_sse2 |
| 952 | +%assign cpuflags_sse3 (1<<8) | cpuflags_sse2 |
| 953 | +%assign cpuflags_ssse3 (1<<9) | cpuflags_sse3 |
| 954 | +%assign cpuflags_sse4 (1<<10) | cpuflags_ssse3 |
| 955 | +%assign cpuflags_sse42 (1<<11) | cpuflags_sse4 |
| 956 | +%assign cpuflags_aesni (1<<12) | cpuflags_sse42 |
| 957 | +%assign cpuflags_clmul (1<<13) | cpuflags_sse42 |
| 958 | +%assign cpuflags_gfni (1<<14) | cpuflags_aesni|cpuflags_clmul |
| 959 | +%assign cpuflags_avx (1<<15) | cpuflags_sse42 |
| 960 | +%assign cpuflags_xop (1<<16) | cpuflags_avx |
| 961 | +%assign cpuflags_fma4 (1<<17) | cpuflags_avx |
| 962 | +%assign cpuflags_fma3 (1<<18) | cpuflags_avx |
| 963 | +%assign cpuflags_bmi1 (1<<19) | cpuflags_avx|cpuflags_lzcnt |
| 964 | +%assign cpuflags_bmi2 (1<<20) | cpuflags_bmi1 |
| 965 | +%assign cpuflags_avx2 (1<<21) | cpuflags_fma3|cpuflags_bmi2 |
| 966 | +%assign cpuflags_avx512 (1<<22) | cpuflags_avx2 ; F, CD, BW, DQ, VL |
| 967 | +%assign cpuflags_avx512icl (1<<23) | cpuflags_avx512|cpuflags_gfni ; VNNI, IFMA, VBMI, VBMI2, VPOPCNTDQ, BITALG, VAES, VPCLMULQDQ |
| 968 | + |
| 969 | +%assign cpuflags_cache32 (1<<24) |
| 970 | +%assign cpuflags_cache64 (1<<25) |
| 971 | +%assign cpuflags_aligned (1<<26) ; not a cpu feature, but a function variant |
| 972 | +%assign cpuflags_atom (1<<27) |
| 857 | 973 | |
| 858 | 974 | ; Returns a boolean value expressing whether or not the specified cpuflag is enabled. |
| 859 | 975 | %define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1) |
| @@ -895,9 +1011,17 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 35 unmodified lines |
| 895 | 1011 | %endif |
| 896 | 1012 | |
| 897 | 1013 | %if ARCH_X86_64 || cpuflag(sse2) |
| 898 | | − CPUNOP amdnop |
| 1014 | + %ifdef __NASM_VERSION_ID__ |
| 1015 | + ALIGNMODE p6 |
| 1016 | + %else |
| 1017 | + CPU amdnop |
| 1018 | + %endif |
| 899 | 1019 | %else |
| 900 | | − CPUNOP basicnop |
| 1020 | + %ifdef __NASM_VERSION_ID__ |
| 1021 | + ALIGNMODE nop |
| 1022 | + %else |
| 1023 | + CPU basicnop |
| 1024 | + %endif |
| 901 | 1025 | %endif |
| 902 | 1026 | %endmacro |
| 903 | 1027 | |
| @@ -971,7 +1095,7 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 67 unmodified lines |
| 971 | 1095 | %endmacro |
| 972 | 1096 | |
| 973 | 1097 | %macro INIT_XMM 0-1+ |
| 974 | | − %assign avx_enabled 0 |
| 1098 | + %assign avx_enabled FORCE_VEX_ENCODING |
| 975 | 1099 | %define RESET_MM_PERMUTATION INIT_XMM %1 |
| 976 | 1100 | %define mmsize 16 |
| 977 | 1101 | %define mova movdqa |
| @@ -983,6 +1107,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 5 unmodified lines |
| 983 | 1107 | %if WIN64 |
| 984 | 1108 | AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers |
| 985 | 1109 | %endif |
| 1110 | + %xdefine bcstw 1to8 |
| 1111 | + %xdefine bcstd 1to4 |
| 1112 | + %xdefine bcstq 1to2 |
| 986 | 1113 | %endmacro |
| 987 | 1114 | |
| 988 | 1115 | %macro INIT_YMM 0-1+ |
| @@ -996,6 +1123,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 7 unmodified lines |
| 996 | 1123 | INIT_CPUFLAGS %1 |
| 997 | 1124 | DEFINE_MMREGS ymm |
| 998 | 1125 | AVX512_MM_PERMUTATION |
| 1126 | + %xdefine bcstw 1to16 |
| 1127 | + %xdefine bcstd 1to8 |
| 1128 | + %xdefine bcstq 1to4 |
| 999 | 1129 | %endmacro |
| 1000 | 1130 | |
| 1001 | 1131 | %macro INIT_ZMM 0-1+ |
| @@ -1009,6 +1139,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 7 unmodified lines |
| 1009 | 1139 | INIT_CPUFLAGS %1 |
| 1010 | 1140 | DEFINE_MMREGS zmm |
| 1011 | 1141 | AVX512_MM_PERMUTATION |
| 1142 | + %xdefine bcstw 1to32 |
| 1143 | + %xdefine bcstd 1to16 |
| 1144 | + %xdefine bcstq 1to8 |
| 1012 | 1145 | %endmacro |
| 1013 | 1146 | |
| 1014 | 1147 | INIT_XMM |
| @@ -1106,19 +1239,32 @@ INIT_XMM · 91 unmodified lines |
| 1106 | 1239 | %endif |
| 1107 | 1240 | %assign %%i 0 |
| 1108 | 1241 | %rep num_mmregs |
| 1109 | | − CAT_XDEFINE %%f, %%i, m %+ %%i |
| 1242 | + %xdefine %%tmp m %+ %%i |
| 1243 | + CAT_XDEFINE %%f, %%i, regnumof %+ %%tmp |
| 1110 | 1244 | %assign %%i %%i+1 |
| 1111 | 1245 | %endrep |
| 1112 | 1246 | %endmacro |
| 1113 | 1247 | |
| 1114 | | −%macro LOAD_MM_PERMUTATION 1 ; name to load from |
| 1115 | | − %ifdef %1_m0 |
| 1248 | +%macro LOAD_MM_PERMUTATION 0-1 ; name to load from |
| 1249 | + %if %0 |
| 1250 | + %xdefine %%f %1_m |
| 1251 | + %else |
| 1252 | + %xdefine %%f current_function %+ _m |
| 1253 | + %endif |
| 1254 | + %xdefine %%tmp %%f %+ 0 |
| 1255 | + %ifnum %%tmp |
| 1256 | + DEFINE_MMREGS mmtype |
| 1116 | 1257 | %assign %%i 0 |
| 1117 | 1258 | %rep num_mmregs |
| 1118 | | − CAT_XDEFINE m, %%i, %1_m %+ %%i |
| 1119 | | − CAT_XDEFINE nn, m %+ %%i, %%i |
| 1259 | + %xdefine %%tmp %%f %+ %%i |
| 1260 | + CAT_XDEFINE %%m, %%i, m %+ %%tmp |
| 1120 | 1261 | %assign %%i %%i+1 |
| 1121 | 1262 | %endrep |
| 1263 | + %rep num_mmregs |
| 1264 | + %assign %%i %%i-1 |
| 1265 | + CAT_XDEFINE m, %%i, %%m %+ %%i |
| 1266 | + CAT_XDEFINE nn, m %+ %%i, %%i |
| 1267 | + %endrep |
| 1122 | 1268 | %endif |
| 1123 | 1269 | %endmacro |
| 1124 | 1270 | |
| @@ -1224,8 +1370,22 @@ INIT_XMM · 99 unmodified lines |
| 1224 | 1370 | %ifdef cpuname |
| 1225 | 1371 | %if notcpuflag(%2) |
| 1226 | 1372 | %error use of ``%1'' %2 instruction in cpuname function: current_function |
| 1227 | | − %elif cpuflags_%2 < cpuflags_sse && notcpuflag(sse2) && __sizeofreg > 8 |
| 1373 | + %elif %3 == 0 && __sizeofreg == 16 && notcpuflag(sse2) |
| 1228 | 1374 | %error use of ``%1'' sse2 instruction in cpuname function: current_function |
| 1375 | + %elif %3 == 0 && __sizeofreg == 32 && notcpuflag(avx2) |
| 1376 | + %error use of ``%1'' avx2 instruction in cpuname function: current_function |
| 1377 | + %elif __sizeofreg == 16 && notcpuflag(sse) |
| 1378 | + %error use of ``%1'' sse instruction in cpuname function: current_function |
| 1379 | + %elif __sizeofreg == 32 && notcpuflag(avx) |
| 1380 | + %error use of ``%1'' avx instruction in cpuname function: current_function |
| 1381 | + %elif __sizeofreg == 64 && notcpuflag(avx512) |
| 1382 | + %error use of ``%1'' avx512 instruction in cpuname function: current_function |
| 1383 | + %elifidn %1, pextrw ; special case because the base instruction is mmx2, |
| 1384 | + %ifnid %6 ; but sse4 is required for memory operands |
| 1385 | + %if notcpuflag(sse4) |
| 1386 | + %error use of ``%1'' sse4 instruction in cpuname function: current_function |
| 1387 | + %endif |
| 1388 | + %endif |
| 1229 | 1389 | %endif |
| 1230 | 1390 | %endif |
| 1231 | 1391 | %endif |
| @@ -1267,11 +1427,79 @@ INIT_XMM · 35 unmodified lines |
| 1267 | 1427 | %1 %6, __src2 |
| 1268 | 1428 | %endif |
| 1269 | 1429 | %elif %0 >= 9 |
| 1270 | | − __instr %6, %7, %8, %9 |
| 1430 | + %if avx_enabled && __sizeofreg >= 16 && %4 == 1 |
| 1431 | + %ifnnum regnumof%7 |
| 1432 | + %if %3 |
| 1433 | + vmovaps %6, %7 |
| 1434 | + %else |
| 1435 | + vmovdqa %6, %7 |
| 1436 | + %endif |
| 1437 | + __instr %6, %6, %8, %9 |
| 1438 | + %else |
| 1439 | + __instr %6, %7, %8, %9 |
| 1440 | + %endif |
| 1441 | + %else |
| 1442 | + __instr %6, %7, %8, %9 |
| 1443 | + %endif |
| 1271 | 1444 | %elif %0 == 8 |
| 1272 | | − __instr %6, %7, %8 |
| 1445 | + %if avx_enabled && __sizeofreg >= 16 && %4 == 0 |
| 1446 | + %xdefine __src1 %7 |
| 1447 | + %xdefine __src2 %8 |
| 1448 | + %if %5 |
| 1449 | + %ifnum regnumof%7 |
| 1450 | + %ifnum regnumof%8 |
| 1451 | + %if regnumof%7 < 8 && regnumof%8 >= 8 && regnumof%8 < 16 && sizeof%8 <= 32 |
| 1452 | + ; Most VEX-encoded instructions require an additional byte to encode when |
| 1453 | + ; src2 is a high register (e.g. m8..15). If the instruction is commutative |
| 1454 | + ; we can swap src1 and src2 when doing so reduces the instruction length. |
| 1455 | + %xdefine __src1 %8 |
| 1456 | + %xdefine __src2 %7 |
| 1457 | + %endif |
| 1458 | + %endif |
| 1459 | + %elifnum regnumof%8 ; put memory operands in src2 when possible |
| 1460 | + %xdefine __src1 %8 |
| 1461 | + %xdefine __src2 %7 |
| 1462 | + %else |
| 1463 | + %assign __emulate_avx 1 |
| 1464 | + %endif |
| 1465 | + %elifnnum regnumof%7 |
| 1466 | + ; EVEX allows imm8 shift instructions to be used with memory operands, |
| 1467 | + ; but VEX does not. This handles those special cases. |
| 1468 | + %ifnnum %8 |
| 1469 | + %assign __emulate_avx 1 |
| 1470 | + %elif notcpuflag(avx512) |
| 1471 | + %assign __emulate_avx 1 |
| 1472 | + %endif |
| 1473 | + %endif |
| 1474 | + %if __emulate_avx ; a separate load is required |
| 1475 | + %if %3 |
| 1476 | + vmovaps %6, %7 |
| 1477 | + %else |
| 1478 | + vmovdqa %6, %7 |
| 1479 | + %endif |
| 1480 | + __instr %6, %6, %8 |
| 1481 | + %else |
| 1482 | + __instr %6, __src1, __src2 |
| 1483 | + %endif |
| 1484 | + %else |
| 1485 | + __instr %6, %7, %8 |
| 1486 | + %endif |
| 1273 | 1487 | %elif %0 == 7 |
| 1274 | | − __instr %6, %7 |
| 1488 | + %if avx_enabled && __sizeofreg >= 16 && %5 |
| 1489 | + %xdefine __src1 %6 |
| 1490 | + %xdefine __src2 %7 |
| 1491 | + %ifnum regnumof%6 |
| 1492 | + %ifnum regnumof%7 |
| 1493 | + %if regnumof%6 < 8 && regnumof%7 >= 8 && regnumof%7 < 16 && sizeof%7 <= 32 |
| 1494 | + %xdefine __src1 %7 |
| 1495 | + %xdefine __src2 %6 |
| 1496 | + %endif |
| 1497 | + %endif |
| 1498 | + %endif |
| 1499 | + __instr %6, __src1, __src2 |
| 1500 | + %else |
| 1501 | + __instr %6, %7 |
| 1502 | + %endif |
| 1275 | 1503 | %else |
| 1276 | 1504 | __instr %6 |
| 1277 | 1505 | %endif |
| @@ -1318,8 +1546,8 @@ AVX_INSTR andpd, sse2, 1, 0, 1 · 40 unmodified lines |
| 1318 | 1546 | AVX_INSTR andps, sse, 1, 0, 1 |
| 1319 | 1547 | AVX_INSTR blendpd, sse4, 1, 1, 0 |
| 1320 | 1548 | AVX_INSTR blendps, sse4, 1, 1, 0 |
| 1321 | | −AVX_INSTR blendvpd, sse4 ; can't be emulated |
| 1322 | | −AVX_INSTR blendvps, sse4 ; can't be emulated |
| 1549 | +AVX_INSTR blendvpd, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding |
| 1550 | +AVX_INSTR blendvps, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding |
| 1323 | 1551 | AVX_INSTR cmpeqpd, sse2, 1, 0, 1 |
| 1324 | 1552 | AVX_INSTR cmpeqps, sse, 1, 0, 1 |
| 1325 | 1553 | AVX_INSTR cmpeqsd, sse2, 1, 0, 0 |
| @@ -1356,38 +1584,41 @@ AVX_INSTR cmpunordpd, sse2, 1, 0, 1 · 30 unmodified lines |
| 1356 | 1584 | AVX_INSTR cmpunordps, sse, 1, 0, 1 |
| 1357 | 1585 | AVX_INSTR cmpunordsd, sse2, 1, 0, 0 |
| 1358 | 1586 | AVX_INSTR cmpunordss, sse, 1, 0, 0 |
| 1359 | | −AVX_INSTR comisd, sse2 |
| 1360 | | −AVX_INSTR comiss, sse |
| 1361 | | −AVX_INSTR cvtdq2pd, sse2 |
| 1362 | | −AVX_INSTR cvtdq2ps, sse2 |
| 1363 | | −AVX_INSTR cvtpd2dq, sse2 |
| 1364 | | −AVX_INSTR cvtpd2ps, sse2 |
| 1365 | | −AVX_INSTR cvtps2dq, sse2 |
| 1366 | | −AVX_INSTR cvtps2pd, sse2 |
| 1367 | | −AVX_INSTR cvtsd2si, sse2 |
| 1587 | +AVX_INSTR comisd, sse2, 1 |
| 1588 | +AVX_INSTR comiss, sse, 1 |
| 1589 | +AVX_INSTR cvtdq2pd, sse2, 1 |
| 1590 | +AVX_INSTR cvtdq2ps, sse2, 1 |
| 1591 | +AVX_INSTR cvtpd2dq, sse2, 1 |
| 1592 | +AVX_INSTR cvtpd2ps, sse2, 1 |
| 1593 | +AVX_INSTR cvtps2dq, sse2, 1 |
| 1594 | +AVX_INSTR cvtps2pd, sse2, 1 |
| 1595 | +AVX_INSTR cvtsd2si, sse2, 1 |
| 1368 | 1596 | AVX_INSTR cvtsd2ss, sse2, 1, 0, 0 |
| 1369 | 1597 | AVX_INSTR cvtsi2sd, sse2, 1, 0, 0 |
| 1370 | 1598 | AVX_INSTR cvtsi2ss, sse, 1, 0, 0 |
| 1371 | 1599 | AVX_INSTR cvtss2sd, sse2, 1, 0, 0 |
| 1372 | | −AVX_INSTR cvtss2si, sse |
| 1373 | | −AVX_INSTR cvttpd2dq, sse2 |
| 1374 | | −AVX_INSTR cvttps2dq, sse2 |
| 1375 | | −AVX_INSTR cvttsd2si, sse2 |
| 1376 | | −AVX_INSTR cvttss2si, sse |
| 1600 | +AVX_INSTR cvtss2si, sse, 1 |
| 1601 | +AVX_INSTR cvttpd2dq, sse2, 1 |
| 1602 | +AVX_INSTR cvttps2dq, sse2, 1 |
| 1603 | +AVX_INSTR cvttsd2si, sse2, 1 |
| 1604 | +AVX_INSTR cvttss2si, sse, 1 |
| 1377 | 1605 | AVX_INSTR divpd, sse2, 1, 0, 0 |
| 1378 | 1606 | AVX_INSTR divps, sse, 1, 0, 0 |
| 1379 | 1607 | AVX_INSTR divsd, sse2, 1, 0, 0 |
| 1380 | 1608 | AVX_INSTR divss, sse, 1, 0, 0 |
| 1381 | 1609 | AVX_INSTR dppd, sse4, 1, 1, 0 |
| 1382 | 1610 | AVX_INSTR dpps, sse4, 1, 1, 0 |
| 1383 | | −AVX_INSTR extractps, sse4 |
| 1611 | +AVX_INSTR extractps, sse4, 1 |
| 1612 | +AVX_INSTR gf2p8affineinvqb, gfni, 0, 1, 0 |
| 1613 | +AVX_INSTR gf2p8affineqb, gfni, 0, 1, 0 |
| 1614 | +AVX_INSTR gf2p8mulb, gfni, 0, 0, 0 |
| 1384 | 1615 | AVX_INSTR haddpd, sse3, 1, 0, 0 |
| 1385 | 1616 | AVX_INSTR haddps, sse3, 1, 0, 0 |
| 1386 | 1617 | AVX_INSTR hsubpd, sse3, 1, 0, 0 |
| 1387 | 1618 | AVX_INSTR hsubps, sse3, 1, 0, 0 |
| 1388 | 1619 | AVX_INSTR insertps, sse4, 1, 1, 0 |
| 1389 | 1620 | AVX_INSTR lddqu, sse3 |
| 1390 | | −AVX_INSTR ldmxcsr, sse |
| 1621 | +AVX_INSTR ldmxcsr, sse, 1 |
| 1391 | 1622 | AVX_INSTR maskmovdqu, sse2 |
| 1392 | 1623 | AVX_INSTR maxpd, sse2, 1, 0, 1 |
| 1393 | 1624 | AVX_INSTR maxps, sse, 1, 0, 1 |
| @@ -1397,10 +1628,10 @@ AVX_INSTR minpd, sse2, 1, 0, 1 · 3 unmodified lines |
| 1397 | 1628 | AVX_INSTR minps, sse, 1, 0, 1 |
| 1398 | 1629 | AVX_INSTR minsd, sse2, 1, 0, 0 |
| 1399 | 1630 | AVX_INSTR minss, sse, 1, 0, 0 |
| 1400 | | −AVX_INSTR movapd, sse2 |
| 1401 | | −AVX_INSTR movaps, sse |
| 1631 | +AVX_INSTR movapd, sse2, 1 |
| 1632 | +AVX_INSTR movaps, sse, 1 |
| 1402 | 1633 | AVX_INSTR movd, mmx |
| 1403 | | −AVX_INSTR movddup, sse3 |
| 1634 | +AVX_INSTR movddup, sse3, 1 |
| 1404 | 1635 | AVX_INSTR movdqa, sse2 |
| 1405 | 1636 | AVX_INSTR movdqu, sse2 |
| 1406 | 1637 | AVX_INSTR movhlps, sse, 1, 0, 0 |
| @@ -1409,19 +1640,19 @@ AVX_INSTR movhps, sse, 1, 0, 0 · 2 unmodified lines |
| 1409 | 1640 | AVX_INSTR movlhps, sse, 1, 0, 0 |
| 1410 | 1641 | AVX_INSTR movlpd, sse2, 1, 0, 0 |
| 1411 | 1642 | AVX_INSTR movlps, sse, 1, 0, 0 |
| 1412 | | −AVX_INSTR movmskpd, sse2 |
| 1413 | | −AVX_INSTR movmskps, sse |
| 1643 | +AVX_INSTR movmskpd, sse2, 1 |
| 1644 | +AVX_INSTR movmskps, sse, 1 |
| 1414 | 1645 | AVX_INSTR movntdq, sse2 |
| 1415 | 1646 | AVX_INSTR movntdqa, sse4 |
| 1416 | | −AVX_INSTR movntpd, sse2 |
| 1417 | | −AVX_INSTR movntps, sse |
| 1647 | +AVX_INSTR movntpd, sse2, 1 |
| 1648 | +AVX_INSTR movntps, sse, 1 |
| 1418 | 1649 | AVX_INSTR movq, mmx |
| 1419 | 1650 | AVX_INSTR movsd, sse2, 1, 0, 0 |
| 1420 | | −AVX_INSTR movshdup, sse3 |
| 1421 | | −AVX_INSTR movsldup, sse3 |
| 1651 | +AVX_INSTR movshdup, sse3, 1 |
| 1652 | +AVX_INSTR movsldup, sse3, 1 |
| 1422 | 1653 | AVX_INSTR movss, sse, 1, 0, 0 |
| 1423 | | −AVX_INSTR movupd, sse2 |
| 1424 | | −AVX_INSTR movups, sse |
| 1654 | +AVX_INSTR movupd, sse2, 1 |
| 1655 | +AVX_INSTR movups, sse, 1 |
| 1425 | 1656 | AVX_INSTR mpsadbw, sse4, 0, 1, 0 |
| 1426 | 1657 | AVX_INSTR mulpd, sse2, 1, 0, 1 |
| 1427 | 1658 | AVX_INSTR mulps, sse, 1, 0, 1 |
| @@ -1432,90 +1663,90 @@ AVX_INSTR orps, sse, 1, 0, 1 · 4 unmodified lines |
| 1432 | 1663 | AVX_INSTR pabsb, ssse3 |
| 1433 | 1664 | AVX_INSTR pabsd, ssse3 |
| 1434 | 1665 | AVX_INSTR pabsw, ssse3 |
| 1435 | | −AVX_INSTR packsswb, mmx, 0, 0, 0 |
| 1436 | 1666 | AVX_INSTR packssdw, mmx, 0, 0, 0 |
| 1437 | | −AVX_INSTR packuswb, mmx, 0, 0, 0 |
| 1667 | +AVX_INSTR packsswb, mmx, 0, 0, 0 |
| 1438 | 1668 | AVX_INSTR packusdw, sse4, 0, 0, 0 |
| 1669 | +AVX_INSTR packuswb, mmx, 0, 0, 0 |
| 1439 | 1670 | AVX_INSTR paddb, mmx, 0, 0, 1 |
| 1440 | | −AVX_INSTR paddw, mmx, 0, 0, 1 |
| 1441 | 1671 | AVX_INSTR paddd, mmx, 0, 0, 1 |
| 1442 | 1672 | AVX_INSTR paddq, sse2, 0, 0, 1 |
| 1443 | 1673 | AVX_INSTR paddsb, mmx, 0, 0, 1 |
| 1444 | 1674 | AVX_INSTR paddsw, mmx, 0, 0, 1 |
| 1445 | 1675 | AVX_INSTR paddusb, mmx, 0, 0, 1 |
| 1446 | 1676 | AVX_INSTR paddusw, mmx, 0, 0, 1 |
| 1677 | +AVX_INSTR paddw, mmx, 0, 0, 1 |
| 1447 | 1678 | AVX_INSTR palignr, ssse3, 0, 1, 0 |
| 1448 | 1679 | AVX_INSTR pand, mmx, 0, 0, 1 |
| 1449 | 1680 | AVX_INSTR pandn, mmx, 0, 0, 0 |
| 1450 | 1681 | AVX_INSTR pavgb, mmx2, 0, 0, 1 |
| 1451 | 1682 | AVX_INSTR pavgw, mmx2, 0, 0, 1 |
| 1452 | | −AVX_INSTR pblendvb, sse4 ; can't be emulated |
| 1683 | +AVX_INSTR pblendvb, sse4, 0, 1, 0 ; last operand must be xmm0 with legacy encoding |
| 1453 | 1684 | AVX_INSTR pblendw, sse4, 0, 1, 0 |
| 1454 | | −AVX_INSTR pclmulqdq, fnord, 0, 1, 0 |
| 1455 | | −AVX_INSTR pclmulhqhqdq, fnord, 0, 0, 0 |
| 1456 | | −AVX_INSTR pclmulhqlqdq, fnord, 0, 0, 0 |
| 1457 | | −AVX_INSTR pclmullqhqdq, fnord, 0, 0, 0 |
| 1458 | | −AVX_INSTR pclmullqlqdq, fnord, 0, 0, 0 |
| 1459 | | −AVX_INSTR pcmpestri, sse42 |
| 1460 | | −AVX_INSTR pcmpestrm, sse42 |
| 1461 | | −AVX_INSTR pcmpistri, sse42 |
| 1462 | | −AVX_INSTR pcmpistrm, sse42 |
| 1685 | +AVX_INSTR pclmulhqhqdq, clmul, 0, 0, 0 |
| 1686 | +AVX_INSTR pclmulhqlqdq, clmul, 0, 0, 0 |
| 1687 | +AVX_INSTR pclmullqhqdq, clmul, 0, 0, 0 |
| 1688 | +AVX_INSTR pclmullqlqdq, clmul, 0, 0, 0 |
| 1689 | +AVX_INSTR pclmulqdq, clmul, 0, 1, 0 |
| 1463 | 1690 | AVX_INSTR pcmpeqb, mmx, 0, 0, 1 |
| 1464 | | −AVX_INSTR pcmpeqw, mmx, 0, 0, 1 |
| 1465 | 1691 | AVX_INSTR pcmpeqd, mmx, 0, 0, 1 |
| 1466 | 1692 | AVX_INSTR pcmpeqq, sse4, 0, 0, 1 |
| 1693 | +AVX_INSTR pcmpeqw, mmx, 0, 0, 1 |
| 1694 | +AVX_INSTR pcmpestri, sse42 |
| 1695 | +AVX_INSTR pcmpestrm, sse42 |
| 1467 | 1696 | AVX_INSTR pcmpgtb, mmx, 0, 0, 0 |
| 1468 | | −AVX_INSTR pcmpgtw, mmx, 0, 0, 0 |
| 1469 | 1697 | AVX_INSTR pcmpgtd, mmx, 0, 0, 0 |
| 1470 | 1698 | AVX_INSTR pcmpgtq, sse42, 0, 0, 0 |
| 1699 | +AVX_INSTR pcmpgtw, mmx, 0, 0, 0 |
| 1700 | +AVX_INSTR pcmpistri, sse42 |
| 1701 | +AVX_INSTR pcmpistrm, sse42 |
| 1471 | 1702 | AVX_INSTR pextrb, sse4 |
| 1472 | 1703 | AVX_INSTR pextrd, sse4 |
| 1473 | 1704 | AVX_INSTR pextrq, sse4 |
| 1474 | 1705 | AVX_INSTR pextrw, mmx2 |
| 1475 | | −AVX_INSTR phaddw, ssse3, 0, 0, 0 |
| 1476 | 1706 | AVX_INSTR phaddd, ssse3, 0, 0, 0 |
| 1477 | 1707 | AVX_INSTR phaddsw, ssse3, 0, 0, 0 |
| 1708 | +AVX_INSTR phaddw, ssse3, 0, 0, 0 |
| 1478 | 1709 | AVX_INSTR phminposuw, sse4 |
| 1479 | | −AVX_INSTR phsubw, ssse3, 0, 0, 0 |
| 1480 | 1710 | AVX_INSTR phsubd, ssse3, 0, 0, 0 |
| 1481 | 1711 | AVX_INSTR phsubsw, ssse3, 0, 0, 0 |
| 1712 | +AVX_INSTR phsubw, ssse3, 0, 0, 0 |
| 1482 | 1713 | AVX_INSTR pinsrb, sse4, 0, 1, 0 |
| 1483 | 1714 | AVX_INSTR pinsrd, sse4, 0, 1, 0 |
| 1484 | 1715 | AVX_INSTR pinsrq, sse4, 0, 1, 0 |
| 1485 | 1716 | AVX_INSTR pinsrw, mmx2, 0, 1, 0 |
| 1486 | | −AVX_INSTR pmaddwd, mmx, 0, 0, 1 |
| 1487 | 1717 | AVX_INSTR pmaddubsw, ssse3, 0, 0, 0 |
| 1718 | +AVX_INSTR pmaddwd, mmx, 0, 0, 1 |
| 1488 | 1719 | AVX_INSTR pmaxsb, sse4, 0, 0, 1 |
| 1489 | | −AVX_INSTR pmaxsw, mmx2, 0, 0, 1 |
| 1490 | 1720 | AVX_INSTR pmaxsd, sse4, 0, 0, 1 |
| 1721 | +AVX_INSTR pmaxsw, mmx2, 0, 0, 1 |
| 1491 | 1722 | AVX_INSTR pmaxub, mmx2, 0, 0, 1 |
| 1492 | | −AVX_INSTR pmaxuw, sse4, 0, 0, 1 |
| 1493 | 1723 | AVX_INSTR pmaxud, sse4, 0, 0, 1 |
| 1724 | +AVX_INSTR pmaxuw, sse4, 0, 0, 1 |
| 1494 | 1725 | AVX_INSTR pminsb, sse4, 0, 0, 1 |
| 1495 | | −AVX_INSTR pminsw, mmx2, 0, 0, 1 |
| 1496 | 1726 | AVX_INSTR pminsd, sse4, 0, 0, 1 |
| 1727 | +AVX_INSTR pminsw, mmx2, 0, 0, 1 |
| 1497 | 1728 | AVX_INSTR pminub, mmx2, 0, 0, 1 |
| 1498 | | −AVX_INSTR pminuw, sse4, 0, 0, 1 |
| 1499 | 1729 | AVX_INSTR pminud, sse4, 0, 0, 1 |
| 1730 | +AVX_INSTR pminuw, sse4, 0, 0, 1 |
| 1500 | 1731 | AVX_INSTR pmovmskb, mmx2 |
| 1501 | | −AVX_INSTR pmovsxbw, sse4 |
| 1502 | 1732 | AVX_INSTR pmovsxbd, sse4 |
| 1503 | 1733 | AVX_INSTR pmovsxbq, sse4 |
| 1734 | +AVX_INSTR pmovsxbw, sse4 |
| 1735 | +AVX_INSTR pmovsxdq, sse4 |
| 1504 | 1736 | AVX_INSTR pmovsxwd, sse4 |
| 1505 | 1737 | AVX_INSTR pmovsxwq, sse4 |
| 1506 | | −AVX_INSTR pmovsxdq, sse4 |
| 1507 | | −AVX_INSTR pmovzxbw, sse4 |
| 1508 | 1738 | AVX_INSTR pmovzxbd, sse4 |
| 1509 | 1739 | AVX_INSTR pmovzxbq, sse4 |
| 1740 | +AVX_INSTR pmovzxbw, sse4 |
| 1741 | +AVX_INSTR pmovzxdq, sse4 |
| 1510 | 1742 | AVX_INSTR pmovzxwd, sse4 |
| 1511 | 1743 | AVX_INSTR pmovzxwq, sse4 |
| 1512 | | −AVX_INSTR pmovzxdq, sse4 |
| 1513 | 1744 | AVX_INSTR pmuldq, sse4, 0, 0, 1 |
| 1514 | 1745 | AVX_INSTR pmulhrsw, ssse3, 0, 0, 1 |
| 1515 | 1746 | AVX_INSTR pmulhuw, mmx2, 0, 0, 1 |
| 1516 | 1747 | AVX_INSTR pmulhw, mmx, 0, 0, 1 |
| 1517 | | −AVX_INSTR pmullw, mmx, 0, 0, 1 |
| 1518 | 1748 | AVX_INSTR pmulld, sse4, 0, 0, 1 |
| 1749 | +AVX_INSTR pmullw, mmx, 0, 0, 1 |
| 1519 | 1750 | AVX_INSTR pmuludq, sse2, 0, 0, 1 |
| 1520 | 1751 | AVX_INSTR por, mmx, 0, 0, 1 |
| 1521 | 1752 | AVX_INSTR psadbw, mmx2, 0, 0, 1 |
| @@ -1524,57 +1755,57 @@ AVX_INSTR pshufd, sse2 · 2 unmodified lines |
| 1524 | 1755 | AVX_INSTR pshufhw, sse2 |
| 1525 | 1756 | AVX_INSTR pshuflw, sse2 |
| 1526 | 1757 | AVX_INSTR psignb, ssse3, 0, 0, 0 |
| 1527 | | −AVX_INSTR psignw, ssse3, 0, 0, 0 |
| 1528 | 1758 | AVX_INSTR psignd, ssse3, 0, 0, 0 |
| 1529 | | −AVX_INSTR psllw, mmx, 0, 0, 0 |
| 1759 | +AVX_INSTR psignw, ssse3, 0, 0, 0 |
| 1530 | 1760 | AVX_INSTR pslld, mmx, 0, 0, 0 |
| 1531 | | −AVX_INSTR psllq, mmx, 0, 0, 0 |
| 1532 | 1761 | AVX_INSTR pslldq, sse2, 0, 0, 0 |
| 1533 | | −AVX_INSTR psraw, mmx, 0, 0, 0 |
| 1762 | +AVX_INSTR psllq, mmx, 0, 0, 0 |
| 1763 | +AVX_INSTR psllw, mmx, 0, 0, 0 |
| 1534 | 1764 | AVX_INSTR psrad, mmx, 0, 0, 0 |
| 1535 | | −AVX_INSTR psrlw, mmx, 0, 0, 0 |
| 1765 | +AVX_INSTR psraw, mmx, 0, 0, 0 |
| 1536 | 1766 | AVX_INSTR psrld, mmx, 0, 0, 0 |
| 1537 | | −AVX_INSTR psrlq, mmx, 0, 0, 0 |
| 1538 | 1767 | AVX_INSTR psrldq, sse2, 0, 0, 0 |
| 1768 | +AVX_INSTR psrlq, mmx, 0, 0, 0 |
| 1769 | +AVX_INSTR psrlw, mmx, 0, 0, 0 |
| 1539 | 1770 | AVX_INSTR psubb, mmx, 0, 0, 0 |
| 1540 | | −AVX_INSTR psubw, mmx, 0, 0, 0 |
| 1541 | 1771 | AVX_INSTR psubd, mmx, 0, 0, 0 |
| 1542 | 1772 | AVX_INSTR psubq, sse2, 0, 0, 0 |
| 1543 | 1773 | AVX_INSTR psubsb, mmx, 0, 0, 0 |
| 1544 | 1774 | AVX_INSTR psubsw, mmx, 0, 0, 0 |
| 1545 | 1775 | AVX_INSTR psubusb, mmx, 0, 0, 0 |
| 1546 | 1776 | AVX_INSTR psubusw, mmx, 0, 0, 0 |
| 1777 | +AVX_INSTR psubw, mmx, 0, 0, 0 |
| 1547 | 1778 | AVX_INSTR ptest, sse4 |
| 1548 | 1779 | AVX_INSTR punpckhbw, mmx, 0, 0, 0 |
| 1549 | | −AVX_INSTR punpckhwd, mmx, 0, 0, 0 |
| 1550 | 1780 | AVX_INSTR punpckhdq, mmx, 0, 0, 0 |
| 1551 | 1781 | AVX_INSTR punpckhqdq, sse2, 0, 0, 0 |
| 1782 | +AVX_INSTR punpckhwd, mmx, 0, 0, 0 |
| 1552 | 1783 | AVX_INSTR punpcklbw, mmx, 0, 0, 0 |
| 1553 | | −AVX_INSTR punpcklwd, mmx, 0, 0, 0 |
| 1554 | 1784 | AVX_INSTR punpckldq, mmx, 0, 0, 0 |
| 1555 | 1785 | AVX_INSTR punpcklqdq, sse2, 0, 0, 0 |
| 1786 | +AVX_INSTR punpcklwd, mmx, 0, 0, 0 |
| 1556 | 1787 | AVX_INSTR pxor, mmx, 0, 0, 1 |
| 1557 | | −AVX_INSTR rcpps, sse |
| 1788 | +AVX_INSTR rcpps, sse, 1 |
| 1558 | 1789 | AVX_INSTR rcpss, sse, 1, 0, 0 |
| 1559 | | −AVX_INSTR roundpd, sse4 |
| 1560 | | −AVX_INSTR roundps, sse4 |
| 1790 | +AVX_INSTR roundpd, sse4, 1 |
| 1791 | +AVX_INSTR roundps, sse4, 1 |
| 1561 | 1792 | AVX_INSTR roundsd, sse4, 1, 1, 0 |
| 1562 | 1793 | AVX_INSTR roundss, sse4, 1, 1, 0 |
| 1563 | | −AVX_INSTR rsqrtps, sse |
| 1794 | +AVX_INSTR rsqrtps, sse, 1 |
| 1564 | 1795 | AVX_INSTR rsqrtss, sse, 1, 0, 0 |
| 1565 | 1796 | AVX_INSTR shufpd, sse2, 1, 1, 0 |
| 1566 | 1797 | AVX_INSTR shufps, sse, 1, 1, 0 |
| 1567 | | −AVX_INSTR sqrtpd, sse2 |
| 1568 | | −AVX_INSTR sqrtps, sse |
| 1798 | +AVX_INSTR sqrtpd, sse2, 1 |
| 1799 | +AVX_INSTR sqrtps, sse, 1 |
| 1569 | 1800 | AVX_INSTR sqrtsd, sse2, 1, 0, 0 |
| 1570 | 1801 | AVX_INSTR sqrtss, sse, 1, 0, 0 |
| 1571 | | −AVX_INSTR stmxcsr, sse |
| 1802 | +AVX_INSTR stmxcsr, sse, 1 |
| 1572 | 1803 | AVX_INSTR subpd, sse2, 1, 0, 0 |
| 1573 | 1804 | AVX_INSTR subps, sse, 1, 0, 0 |
| 1574 | 1805 | AVX_INSTR subsd, sse2, 1, 0, 0 |
| 1575 | 1806 | AVX_INSTR subss, sse, 1, 0, 0 |
| 1576 | | −AVX_INSTR ucomisd, sse2 |
| 1577 | | −AVX_INSTR ucomiss, sse |
| 1807 | +AVX_INSTR ucomisd, sse2, 1 |
| 1808 | +AVX_INSTR ucomiss, sse, 1 |
| 1578 | 1809 | AVX_INSTR unpckhpd, sse2, 1, 0, 0 |
| 1579 | 1810 | AVX_INSTR unpckhps, sse, 1, 0, 0 |
| 1580 | 1811 | AVX_INSTR unpcklpd, sse2, 1, 0, 0 |
| @@ -1584,8 +1815,41 @@ AVX_INSTR xorps, sse, 1, 0, 1 · 3 unmodified lines |
| 1584 | 1815 | |
| 1585 | 1816 | ; 3DNow instructions, for sharing code between AVX, SSE and 3DN |
| 1586 | 1817 | AVX_INSTR pfadd, 3dnow, 1, 0, 1 |
| 1587 | | −AVX_INSTR pfsub, 3dnow, 1, 0, 0 |
| 1588 | 1818 | AVX_INSTR pfmul, 3dnow, 1, 0, 1 |
| 1819 | +AVX_INSTR pfsub, 3dnow, 1, 0, 0 |
| 1820 | + |
| 1821 | +;%1 == instruction |
| 1822 | +;%2 == minimal instruction set |
| 1823 | +%macro GPR_INSTR 2 |
| 1824 | + %macro %1 2-5 fnord, %1, %2 |
| 1825 | + %ifdef cpuname |
| 1826 | + %if notcpuflag(%5) |
| 1827 | + %error use of ``%4'' %5 instruction in cpuname function: current_function |
| 1828 | + %endif |
| 1829 | + %endif |
| 1830 | + %ifidn %3, fnord |
| 1831 | + %4 %1, %2 |
| 1832 | + %else |
| 1833 | + %4 %1, %2, %3 |
| 1834 | + %endif |
| 1835 | + %endmacro |
| 1836 | +%endmacro |
| 1837 | + |
| 1838 | +GPR_INSTR andn, bmi1 |
| 1839 | +GPR_INSTR bextr, bmi1 |
| 1840 | +GPR_INSTR blsi, bmi1 |
| 1841 | +GPR_INSTR blsmsk, bmi1 |
| 1842 | +GPR_INSTR blsr, bmi1 |
| 1843 | +GPR_INSTR bzhi, bmi2 |
| 1844 | +GPR_INSTR crc32, sse42 |
| 1845 | +GPR_INSTR mulx, bmi2 |
| 1846 | +GPR_INSTR pdep, bmi2 |
| 1847 | +GPR_INSTR pext, bmi2 |
| 1848 | +GPR_INSTR popcnt, sse42 |
| 1849 | +GPR_INSTR rorx, bmi2 |
| 1850 | +GPR_INSTR sarx, bmi2 |
| 1851 | +GPR_INSTR shlx, bmi2 |
| 1852 | +GPR_INSTR shrx, bmi2 |
| 1589 | 1853 | |
| 1590 | 1854 | ; base-4 constants for shuffles |
| 1591 | 1855 | %assign i 0 |
| @@ -1618,15 +1882,11 @@ AVX_INSTR pfmul, 3dnow, 1, 0, 1 · 26 unmodified lines |
| 1618 | 1882 | %endmacro |
| 1619 | 1883 | %endmacro |
| 1620 | 1884 | |
| 1621 | | −FMA_INSTR pmacsww, pmullw, paddw |
| 1622 | | −FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation |
| 1623 | | −FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation |
| 1885 | +FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation |
| 1886 | +FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation |
| 1887 | +FMA_INSTR pmacsww, pmullw, paddw |
| 1624 | 1888 | FMA_INSTR pmadcswd, pmaddwd, paddd |
| 1625 | 1889 | |
| 1626 | | −; tzcnt is equivalent to "rep bsf" and is backwards-compatible with bsf. |
| 1627 | | −; This lets us use tzcnt without bumping the yasm version requirement yet. |
| 1628 | | −%define tzcnt rep bsf |
| 1629 | | − |
| 1630 | 1890 | ; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax. |
| 1631 | 1891 | ; FMA3 is only possible if dst is the same as one of the src registers. |
| 1632 | 1892 | ; Either src2 or src3 can be a memory operand. |
| @@ -1687,6 +1947,11 @@ FMA4_INSTR fnmsub, pd, ps, sd, ss · 54 unmodified lines |
| 1687 | 1947 | %assign %%evex_required 1 |
| 1688 | 1948 | %endif |
| 1689 | 1949 | %endif |
| 1950 | + %ifnum regnumof%3 |
| 1951 | + %if regnumof%3 >= 16 || sizeof%3 > 32 |
| 1952 | + %assign %%evex_required 1 |
| 1953 | + %endif |
| 1954 | + %endif |
| 1690 | 1955 | %if %%evex_required |
| 1691 | 1956 | %6 %%args |
| 1692 | 1957 | %else |
| @@ -1711,16 +1976,3 @@ EVEX_INSTR vrcpps, vrcp14ps, 1 ; EVEX versions have higher precision · 18 unmodified lines |
| 1711 | 1976 | EVEX_INSTR vrcpss, vrcp14ss, 1 |
| 1712 | 1977 | EVEX_INSTR vrsqrtps, vrsqrt14ps, 1 |
| 1713 | 1978 | EVEX_INSTR vrsqrtss, vrsqrt14ss, 1 |
| 1714 | | − |
| 1715 | | −; workaround: vpbroadcastq is broken in x86_32 due to a yasm bug (fixed in 1.3.0) |
| 1716 | | −%ifdef __YASM_VER__ |
| 1717 | | − %if __YASM_VERSION_ID__ < 0x01030000 && ARCH_X86_64 == 0 |
| 1718 | | − %macro vpbroadcastq 2 |
| 1719 | | − %if sizeof%1 == 16 |
| 1720 | | − movddup %1, %2 |
| 1721 | | − %else |
| 1722 | | − vbroadcastsd %1, %2 |
| 1723 | | − %endif |
| 1724 | | − %endmacro |
| 1725 | | − %endif |
| 1726 | | −%endif |