| @@ -1,7 +1,7 @@ |
| 1 | ;***************************************************************************** | 1 | ;***************************************************************************** |
| 2 | ;* x86inc.asm: x264asm abstraction layer | 2 | ;* x86inc.asm: x86 abstraction layer |
| 3 | ;***************************************************************************** | 3 | ;***************************************************************************** |
| 4 | ;* Copyright (C) 2005-2018 x264 project | 4 | ;* Copyright (C) 2005-2024 x264 project |
| 5 | ;* | 5 | ;* |
| 6 | ;* Authors: Loren Merritt <lorenm@u.washington.edu> | 6 | ;* Authors: Loren Merritt <lorenm@u.washington.edu> |
| 7 | ;* Henrik Gramner <henrik@gramner.com> | 7 | ;* Henrik Gramner <henrik@gramner.com> |
| @@ -21,21 +21,14 @@ · 13 unmodified lines |
| 21 | ;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. | 21 | ;* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. |
| 22 | ;***************************************************************************** | 22 | ;***************************************************************************** |
| 23 | | 23 | |
| 24 | ; This is a header file for the x264ASM assembly language, which uses | 24 | ; This is a header file for the x86inc.asm assembly language, which uses |
| 25 | ; NASM/YASM syntax combined with a large number of macros to provide easy | 25 | ; NASM/YASM syntax combined with a large number of macros to provide easy |
| 26 | ; abstraction between different calling conventions (x86_32, win64, linux64). | 26 | ; abstraction between different calling conventions (x86_32, win64, linux64). |
| 27 | ; It also has various other useful features to simplify writing the kind of | 27 | ; It also has various other useful features to simplify writing the kind of |
| 28 | ; DSP functions that are most often used in x264. | 28 | ; DSP functions that are most often used. |
| 29 | | | |
| 30 | ; Unlike the rest of x264, this file is available under an ISC license, as it | | |
| 31 | ; has significant usefulness outside of x264 and we want it to be available | | |
| 32 | ; to the largest audience possible. Of course, if you modify it for your own | | |
| 33 | ; purposes to add a new feature, we strongly encourage contributing a patch | | |
| 34 | ; as this feature might be useful for others as well. Send patches or ideas | | |
| 35 | ; to x264-devel@videolan.org . | | |
| 36 | | 29 | |
| 37 | %ifndef private_prefix | 30 | %ifndef private_prefix |
| 38 | %define private_prefix x264 | 31 | %error private_prefix not defined |
| 39 | %endif | 32 | %endif |
| 40 | | 33 | |
| 41 | %ifndef public_prefix | 34 | %ifndef public_prefix |
| @@ -68,12 +61,19 @@ · 26 unmodified lines |
| 68 | %endif | 61 | %endif |
| 69 | | 62 | |
| 70 | %define FORMAT_ELF 0 | 63 | %define FORMAT_ELF 0 |
| | 64 | %define FORMAT_MACHO 0 |
| 71 | %ifidn __OUTPUT_FORMAT__,elf | 65 | %ifidn __OUTPUT_FORMAT__,elf |
| 72 | %define FORMAT_ELF 1 | 66 | %define FORMAT_ELF 1 |
| 73 | %elifidn __OUTPUT_FORMAT__,elf32 | 67 | %elifidn __OUTPUT_FORMAT__,elf32 |
| 74 | %define FORMAT_ELF 1 | 68 | %define FORMAT_ELF 1 |
| 75 | %elifidn __OUTPUT_FORMAT__,elf64 | 69 | %elifidn __OUTPUT_FORMAT__,elf64 |
| 76 | %define FORMAT_ELF 1 | 70 | %define FORMAT_ELF 1 |
| | 71 | %elifidn __OUTPUT_FORMAT__,macho |
| | 72 | %define FORMAT_MACHO 1 |
| | 73 | %elifidn __OUTPUT_FORMAT__,macho32 |
| | 74 | %define FORMAT_MACHO 1 |
| | 75 | %elifidn __OUTPUT_FORMAT__,macho64 |
| | 76 | %define FORMAT_MACHO 1 |
| 77 | %endif | 77 | %endif |
| 78 | | 78 | |
| 79 | %ifdef PREFIX | 79 | %ifdef PREFIX |
| @@ -82,6 +82,11 @@ · 2 unmodified lines |
| 82 | %define mangle(x) x | 82 | %define mangle(x) x |
| 83 | %endif | 83 | %endif |
| 84 | | 84 | |
| | 85 | ; Use VEX-encoding even in non-AVX functions |
| | 86 | %ifndef FORCE_VEX_ENCODING |
| | 87 | %define FORCE_VEX_ENCODING 0 |
| | 88 | %endif |
| | 89 | |
| 85 | ; aout does not support align= | 90 | ; aout does not support align= |
| 86 | ; NOTE: This section is out of sync with x264, in order to | 91 | ; NOTE: This section is out of sync with x264, in order to |
| 87 | ; keep supporting OS/2. | 92 | ; keep supporting OS/2. |
| @@ -99,28 +104,27 @@ · 11 unmodified lines |
| 99 | %endif | 104 | %endif |
| 100 | %endmacro | 105 | %endmacro |
| 101 | | 106 | |
| 102 | %if WIN64 | 107 | %if ARCH_X86_64 |
| 103 | %define PIC | 108 | %define PIC 1 ; always use PIC on x86-64 |
| 104 | %elif ARCH_X86_64 == 0 | | |
| 105 | ; x86_32 doesn't require PIC. | | |
| 106 | ; Some distros prefer shared objects to be PIC, but nothing breaks if | | |
| 107 | ; the code contains a few textrels, so we'll skip that complexity. | | |
| 108 | %undef PIC | | |
| 109 | %endif | | |
| 110 | %ifdef PIC | | |
| 111 | default rel | 109 | default rel |
| | 110 | %elifidn __OUTPUT_FORMAT__,win32 |
| | 111 | %define PIC 0 ; PIC isn't used on 32-bit Windows |
| | 112 | %elifndef PIC |
| | 113 | %define PIC 0 |
| 112 | %endif | 114 | %endif |
| 113 | | 115 | |
| 114 | %macro CPUNOP 1 | 116 | %define HAVE_PRIVATE_EXTERN 1 |
| 115 | %if HAVE_CPUNOP | 117 | %ifdef __NASM_VERSION_ID__ |
| 116 | CPU %1 | 118 | %use smartalign |
| | 119 | %if __NASM_VERSION_ID__ < 0x020e0000 ; 2.14 |
| | 120 | %define HAVE_PRIVATE_EXTERN 0 |
| 117 | %endif | 121 | %endif |
| 118 | %endmacro | 122 | %endif |
| 119 | | 123 | |
| 120 | ; Macros to eliminate most code duplication between x86_32 and x86_64: | 124 | ; Macros to eliminate most code duplication between x86_32 and x86_64: |
| 121 | ; Currently this works only for leaf functions which load all their arguments | 125 | ; Currently this works only for leaf functions which load all their arguments |
| 122 | ; into registers at the start, and make no other use of the stack. Luckily that | 126 | ; into registers at the start, and make no other use of the stack. Luckily that |
| 123 | ; covers most of x264's asm. | 127 | ; covers most use cases. |
| 124 | | 128 | |
| 125 | ; PROLOGUE: | 129 | ; PROLOGUE: |
| 126 | ; %1 = number of arguments. loads them from stack if needed. | 130 | ; %1 = number of arguments. loads them from stack if needed. |
| @@ -232,6 +236,18 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 105 unmodified lines |
| 232 | %define gprsize 4 | 236 | %define gprsize 4 |
| 233 | %endif | 237 | %endif |
| 234 | | 238 | |
| | 239 | %macro LEA 2 |
| | 240 | %if ARCH_X86_64 |
| | 241 | lea %1, [%2] |
| | 242 | %elif PIC |
| | 243 | call $+5 ; special-cased to not affect the RSB on most CPU:s |
| | 244 | pop %1 |
| | 245 | add %1, (%2)-$+1 |
| | 246 | %else |
| | 247 | mov %1, %2 |
| | 248 | %endif |
| | 249 | %endmacro |
| | 250 | |
| 235 | ; Repeats an instruction/operation for multiple arguments. | 251 | ; Repeats an instruction/operation for multiple arguments. |
| 236 | ; Example usage: "REPX {psrlw x, 8}, m0, m1, m2, m3" | 252 | ; Example usage: "REPX {psrlw x, 8}, m0, m1, m2, m3" |
| 237 | %macro REPX 2-* ; operation, args | 253 | %macro REPX 2-* ; operation, args |
| @@ -303,6 +319,10 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 65 unmodified lines |
| 303 | %endif | 319 | %endif |
| 304 | %endmacro | 320 | %endmacro |
| 305 | | 321 | |
| | 322 | %if ARCH_X86_64 == 0 |
| | 323 | %define movsxd movifnidn |
| | 324 | %endif |
| | 325 | |
| 306 | %macro movsxdifnidn 2 | 326 | %macro movsxdifnidn 2 |
| 307 | %ifnidn %1, %2 | 327 | %ifnidn %1, %2 |
| 308 | movsxd %1, %2 | 328 | movsxd %1, %2 |
| @@ -354,7 +374,46 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 45 unmodified lines |
| 354 | %define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512))) | 374 | %define vzeroupper_required (mmsize > 16 && (ARCH_X86_64 == 0 || xmm_regs_used > 16 || notcpuflag(avx512))) |
| 355 | %define high_mm_regs (16*cpuflag(avx512)) | 375 | %define high_mm_regs (16*cpuflag(avx512)) |
| 356 | | 376 | |
| 357 | %macro ALLOC_STACK 1-2 0 ; stack_size, n_xmm_regs (for win64 only) | 377 | ; Large stack allocations on Windows need to use stack probing in order |
| | 378 | ; to guarantee that all stack memory is committed before accessing it. |
| | 379 | ; This is done by ensuring that the guard page(s) at the end of the |
| | 380 | ; currently committed pages are touched prior to any pages beyond that. |
| | 381 | %if WIN64 |
| | 382 | %assign STACK_PROBE_SIZE 8192 |
| | 383 | %elifidn __OUTPUT_FORMAT__, win32 |
| | 384 | %assign STACK_PROBE_SIZE 4096 |
| | 385 | %else |
| | 386 | %assign STACK_PROBE_SIZE 0 |
| | 387 | %endif |
| | 388 | |
| | 389 | %macro PROBE_STACK 1 ; stack_size |
| | 390 | %if STACK_PROBE_SIZE |
| | 391 | %assign %%i STACK_PROBE_SIZE |
| | 392 | %rep %1 / STACK_PROBE_SIZE |
| | 393 | mov eax, [rsp-%%i] |
| | 394 | %assign %%i %%i+STACK_PROBE_SIZE |
| | 395 | %endrep |
| | 396 | %endif |
| | 397 | %endmacro |
| | 398 | |
| | 399 | %macro RESET_STACK_STATE 0 |
| | 400 | %ifidn rstk, rsp |
| | 401 | %assign stack_offset stack_offset - stack_size_padded |
| | 402 | %else |
| | 403 | %xdefine rstk rsp |
| | 404 | %endif |
| | 405 | %assign stack_size 0 |
| | 406 | %assign stack_size_padded 0 |
| | 407 | %assign xmm_regs_used 0 |
| | 408 | %endmacro |
| | 409 | |
| | 410 | %macro ALLOC_STACK 0-2 0, 0 ; stack_size, n_xmm_regs |
| | 411 | RESET_STACK_STATE |
| | 412 | %ifnum %2 |
| | 413 | %if mmsize != 8 |
| | 414 | %assign xmm_regs_used %2 |
| | 415 | %endif |
| | 416 | %endif |
| 358 | %ifnum %1 | 417 | %ifnum %1 |
| 359 | %if %1 != 0 | 418 | %if %1 != 0 |
| 360 | %assign %%pad 0 | 419 | %assign %%pad 0 |
| @@ -364,16 +423,14 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 3 unmodified lines |
| 364 | %endif | 423 | %endif |
| 365 | %if WIN64 | 424 | %if WIN64 |
| 366 | %assign %%pad %%pad + 32 ; shadow space | 425 | %assign %%pad %%pad + 32 ; shadow space |
| 367 | %if mmsize != 8 | 426 | %if xmm_regs_used > 8 |
| 368 | %assign xmm_regs_used %2 | 427 | %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers |
| 369 | %if xmm_regs_used > 8 | | |
| 370 | %assign %%pad %%pad + (xmm_regs_used-8)*16 ; callee-saved xmm registers | | |
| 371 | %endif | | |
| 372 | %endif | 428 | %endif |
| 373 | %endif | 429 | %endif |
| 374 | %if required_stack_alignment <= STACK_ALIGNMENT | 430 | %if required_stack_alignment <= STACK_ALIGNMENT |
| 375 | ; maintain the current stack alignment | 431 | ; maintain the current stack alignment |
| 376 | %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) | 432 | %assign stack_size_padded stack_size + %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) |
| | 433 | PROBE_STACK stack_size_padded |
| 377 | SUB rsp, stack_size_padded | 434 | SUB rsp, stack_size_padded |
| 378 | %else | 435 | %else |
| 379 | %assign %%reg_num (regs_used - 1) | 436 | %assign %%reg_num (regs_used - 1) |
| @@ -389,6 +446,7 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 9 unmodified lines |
| 389 | %xdefine rstkm rstk | 446 | %xdefine rstkm rstk |
| 390 | %endif | 447 | %endif |
| 391 | %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1)) | 448 | %assign stack_size_padded stack_size + ((%%pad + required_stack_alignment-1) & ~(required_stack_alignment-1)) |
| | 449 | PROBE_STACK stack_size_padded |
| 392 | mov rstk, rsp | 450 | mov rstk, rsp |
| 393 | and rsp, ~(required_stack_alignment-1) | 451 | and rsp, ~(required_stack_alignment-1) |
| 394 | sub rsp, stack_size_padded | 452 | sub rsp, stack_size_padded |
| @@ -399,7 +457,7 @@ DECLARE_REG_TMP_SIZE 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14 · 4 unmodified lines |
| 399 | %endif | 457 | %endif |
| 400 | %endmacro | 458 | %endmacro |
| 401 | | 459 | |
| 402 | %macro SETUP_STACK_POINTER 1 | 460 | %macro SETUP_STACK_POINTER 0-1 0 |
| 403 | %ifnum %1 | 461 | %ifnum %1 |
| 404 | %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT | 462 | %if %1 != 0 && required_stack_alignment > STACK_ALIGNMENT |
| 405 | %if %1 > 0 | 463 | %if %1 > 0 |
| @@ -462,35 +520,62 @@ DECLARE_REG 14, R13, 120 · 56 unmodified lines |
| 462 | %endif | 520 | %endif |
| 463 | %endmacro | 521 | %endmacro |
| 464 | | 522 | |
| 465 | %macro WIN64_PUSH_XMM 0 | 523 | ; Push XMM registers to the stack. If no argument is specified all used register |
| 466 | ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. | 524 | ; will be pushed, otherwise only push previously unpushed registers. |
| 467 | %if xmm_regs_used > 6 + high_mm_regs | 525 | %macro WIN64_PUSH_XMM 0-2 ; new_xmm_regs_used, xmm_regs_pushed |
| 468 | movaps [rstk + stack_offset + 8], xmm6 | 526 | %if mmsize != 8 |
| 469 | %endif | 527 | %if %0 == 2 |
| 470 | %if xmm_regs_used > 7 + high_mm_regs | 528 | %assign %%pushed %2 |
| 471 | movaps [rstk + stack_offset + 24], xmm7 | 529 | %assign xmm_regs_used %1 |
| 472 | %endif | 530 | %elif %0 == 1 |
| 473 | %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8 | 531 | %assign %%pushed xmm_regs_used |
| 474 | %if %%xmm_regs_on_stack > 0 | 532 | %assign xmm_regs_used %1 |
| 475 | %assign %%i 8 | 533 | %else |
| 476 | %rep %%xmm_regs_on_stack | 534 | %assign %%pushed 0 |
| 477 | movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i | 535 | %endif |
| 478 | %assign %%i %%i+1 | 536 | ; Use the shadow space to store XMM6 and XMM7, the rest needs stack space allocated. |
| 479 | %endrep | 537 | %if %%pushed <= 6 + high_mm_regs && xmm_regs_used > 6 + high_mm_regs |
| | 538 | movaps [rstk + stack_offset + 8], xmm6 |
| | 539 | %endif |
| | 540 | %if %%pushed <= 7 + high_mm_regs && xmm_regs_used > 7 + high_mm_regs |
| | 541 | movaps [rstk + stack_offset + 24], xmm7 |
| | 542 | %endif |
| | 543 | %assign %%pushed %%pushed - high_mm_regs - 8 |
| | 544 | %if %%pushed < 0 |
| | 545 | %assign %%pushed 0 |
| | 546 | %endif |
| | 547 | %assign %%regs_to_push xmm_regs_used - %%pushed - high_mm_regs - 8 |
| | 548 | %if %%regs_to_push > 0 |
| | 549 | ASSERT (%%regs_to_push + %%pushed) * 16 <= stack_size_padded - stack_size - 32 |
| | 550 | %assign %%i %%pushed + 8 |
| | 551 | %rep %%regs_to_push |
| | 552 | movaps [rsp + (%%i-8)*16 + stack_size + 32], xmm %+ %%i |
| | 553 | %assign %%i %%i+1 |
| | 554 | %endrep |
| | 555 | %endif |
| 480 | %endif | 556 | %endif |
| 481 | %endmacro | 557 | %endmacro |
| 482 | | 558 | |
| 483 | %macro WIN64_SPILL_XMM 1 | 559 | ; Allocated stack space for XMM registers and push all, or a subset, of those |
| 484 | %assign xmm_regs_used %1 | 560 | %macro WIN64_SPILL_XMM 1-2 ; xmm_regs_used, xmm_regs_reserved |
| 485 | ASSERT xmm_regs_used <= 16 + high_mm_regs | 561 | RESET_STACK_STATE |
| 486 | %assign %%xmm_regs_on_stack xmm_regs_used - high_mm_regs - 8 | 562 | %if mmsize != 8 |
| 487 | %if %%xmm_regs_on_stack > 0 | 563 | %assign xmm_regs_used %1 |
| 488 | ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack. | 564 | ASSERT xmm_regs_used <= 16 + high_mm_regs |
| 489 | %assign %%pad %%xmm_regs_on_stack*16 + 32 | 565 | %if %0 == 2 |
| 490 | %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) | 566 | ASSERT %2 >= %1 |
| 491 | SUB rsp, stack_size_padded | 567 | %assign %%xmm_regs_on_stack %2 - high_mm_regs - 8 |
| | 568 | %else |
| | 569 | %assign %%xmm_regs_on_stack %1 - high_mm_regs - 8 |
| | 570 | %endif |
| | 571 | %if %%xmm_regs_on_stack > 0 |
| | 572 | ; Allocate stack space for callee-saved xmm registers plus shadow space and align the stack. |
| | 573 | %assign %%pad %%xmm_regs_on_stack*16 + 32 |
| | 574 | %assign stack_size_padded %%pad + ((-%%pad-stack_offset-gprsize) & (STACK_ALIGNMENT-1)) |
| | 575 | SUB rsp, stack_size_padded |
| | 576 | %endif |
| | 577 | WIN64_PUSH_XMM |
| 492 | %endif | 578 | %endif |
| 493 | WIN64_PUSH_XMM | | |
| 494 | %endmacro | 579 | %endmacro |
| 495 | | 580 | |
| 496 | %macro WIN64_RESTORE_XMM_INTERNAL 0 | 581 | %macro WIN64_RESTORE_XMM_INTERNAL 0 |
| @@ -521,9 +606,7 @@ DECLARE_REG 14, R13, 120 · 24 unmodified lines |
| 521 | | 606 | |
| 522 | %macro WIN64_RESTORE_XMM 0 | 607 | %macro WIN64_RESTORE_XMM 0 |
| 523 | WIN64_RESTORE_XMM_INTERNAL | 608 | WIN64_RESTORE_XMM_INTERNAL |
| 524 | %assign stack_offset (stack_offset-stack_size_padded) | 609 | RESET_STACK_STATE |
| 525 | %assign stack_size_padded 0 | | |
| 526 | %assign xmm_regs_used 0 | | |
| 527 | %endmacro | 610 | %endmacro |
| 528 | | 611 | |
| 529 | %define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs | 612 | %define has_epilogue regs_used > 7 || stack_size > 0 || vzeroupper_required || xmm_regs_used > 6+high_mm_regs |
| @@ -558,12 +641,11 @@ DECLARE_REG 14, R13, 72 · 28 unmodified lines |
| 558 | %macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... | 641 | %macro PROLOGUE 2-5+ 0, 0 ; #args, #regs, #xmm_regs, [stack_size,] arg_names... |
| 559 | %assign num_args %1 | 642 | %assign num_args %1 |
| 560 | %assign regs_used %2 | 643 | %assign regs_used %2 |
| 561 | %assign xmm_regs_used %3 | | |
| 562 | ASSERT regs_used >= num_args | 644 | ASSERT regs_used >= num_args |
| 563 | SETUP_STACK_POINTER %4 | 645 | SETUP_STACK_POINTER %4 |
| 564 | ASSERT regs_used <= 15 | 646 | ASSERT regs_used <= 15 |
| 565 | PUSH_IF_USED 9, 10, 11, 12, 13, 14 | 647 | PUSH_IF_USED 9, 10, 11, 12, 13, 14 |
| 566 | ALLOC_STACK %4 | 648 | ALLOC_STACK %4, %3 |
| 567 | LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 | 649 | LOAD_IF_USED 6, 7, 8, 9, 10, 11, 12, 13, 14 |
| 568 | %if %0 > 4 | 650 | %if %0 > 4 |
| 569 | %ifnum %4 | 651 | %ifnum %4 |
| @@ -627,7 +709,7 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 57 unmodified lines |
| 627 | SETUP_STACK_POINTER %4 | 709 | SETUP_STACK_POINTER %4 |
| 628 | ASSERT regs_used <= 7 | 710 | ASSERT regs_used <= 7 |
| 629 | PUSH_IF_USED 3, 4, 5, 6 | 711 | PUSH_IF_USED 3, 4, 5, 6 |
| 630 | ALLOC_STACK %4 | 712 | ALLOC_STACK %4, %3 |
| 631 | LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 | 713 | LOAD_IF_USED 0, 1, 2, 3, 4, 5, 6 |
| 632 | %if %0 > 4 | 714 | %if %0 > 4 |
| 633 | %ifnum %4 | 715 | %ifnum %4 |
| @@ -660,11 +742,19 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 26 unmodified lines |
| 660 | %endif ;====================================================================== | 742 | %endif ;====================================================================== |
| 661 | | 743 | |
| 662 | %if WIN64 == 0 | 744 | %if WIN64 == 0 |
| 663 | %macro WIN64_SPILL_XMM 1 | 745 | %macro WIN64_SPILL_XMM 1-2 |
| | 746 | RESET_STACK_STATE |
| | 747 | %if mmsize != 8 |
| | 748 | %assign xmm_regs_used %1 |
| | 749 | %endif |
| 664 | %endmacro | 750 | %endmacro |
| 665 | %macro WIN64_RESTORE_XMM 0 | 751 | %macro WIN64_RESTORE_XMM 0 |
| | 752 | RESET_STACK_STATE |
| 666 | %endmacro | 753 | %endmacro |
| 667 | %macro WIN64_PUSH_XMM 0 | 754 | %macro WIN64_PUSH_XMM 0-2 |
| | 755 | %if mmsize != 8 && %0 >= 1 |
| | 756 | %assign xmm_regs_used %1 |
| | 757 | %endif |
| 668 | %endmacro | 758 | %endmacro |
| 669 | %endif | 759 | %endif |
| 670 | | 760 | |
| @@ -705,7 +795,7 @@ DECLARE_ARG 7, 8, 9, 10, 11, 12, 13, 14 · 34 unmodified lines |
| 705 | | 795 | |
| 706 | BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp | 796 | BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, jna, jnae, jb, jbe, jnb, jnbe, jc, jnc, js, jns, jo, jno, jp, jnp |
| 707 | | 797 | |
| 708 | %macro TAIL_CALL 2 ; callee, is_nonadjacent | 798 | %macro TAIL_CALL 1-2 1 ; callee, is_nonadjacent |
| 709 | %if has_epilogue | 799 | %if has_epilogue |
| 710 | call %1 | 800 | call %1 |
| 711 | RET | 801 | RET |
| @@ -735,22 +825,25 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 23 unmodified lines |
| 735 | %endmacro | 825 | %endmacro |
| 736 | %macro cglobal_internal 2-3+ | 826 | %macro cglobal_internal 2-3+ |
| 737 | annotate_function_size | 827 | annotate_function_size |
| 738 | %if %1 | | |
| 739 | %xdefine %%FUNCTION_PREFIX private_prefix | | |
| 740 | %xdefine %%VISIBILITY hidden | | |
| 741 | %else | | |
| 742 | %xdefine %%FUNCTION_PREFIX public_prefix | | |
| 743 | %xdefine %%VISIBILITY | | |
| 744 | %endif | | |
| 745 | %ifndef cglobaled_%2 | 828 | %ifndef cglobaled_%2 |
| 746 | %xdefine %2 mangle(%%FUNCTION_PREFIX %+ _ %+ %2) | 829 | %if %1 |
| | 830 | %xdefine %2 mangle(private_prefix %+ _ %+ %2) |
| | 831 | %else |
| | 832 | %xdefine %2 mangle(public_prefix %+ _ %+ %2) |
| | 833 | %endif |
| 747 | %xdefine %2.skip_prologue %2 %+ .skip_prologue | 834 | %xdefine %2.skip_prologue %2 %+ .skip_prologue |
| 748 | CAT_XDEFINE cglobaled_, %2, 1 | 835 | CAT_XDEFINE cglobaled_, %2, 1 |
| 749 | %endif | 836 | %endif |
| 750 | %xdefine current_function %2 | 837 | %xdefine current_function %2 |
| 751 | %xdefine current_function_section __SECT__ | 838 | %xdefine current_function_section __SECT__ |
| 752 | %if FORMAT_ELF | 839 | %if FORMAT_ELF |
| 753 | global %2:function %%VISIBILITY | 840 | %if %1 |
| | 841 | global %2:function hidden |
| | 842 | %else |
| | 843 | global %2:function |
| | 844 | %endif |
| | 845 | %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN && %1 |
| | 846 | global %2:private_extern |
| 754 | %else | 847 | %else |
| 755 | global %2 | 848 | global %2 |
| 756 | %endif | 849 | %endif |
| @@ -771,6 +864,8 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 14 unmodified lines |
| 771 | %macro cglobal_label 1 | 864 | %macro cglobal_label 1 |
| 772 | %if FORMAT_ELF | 865 | %if FORMAT_ELF |
| 773 | global current_function %+ %1:function hidden | 866 | global current_function %+ %1:function hidden |
| | 867 | %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN |
| | 868 | global current_function %+ %1:private_extern |
| 774 | %else | 869 | %else |
| 775 | global current_function %+ %1 | 870 | global current_function %+ %1 |
| 776 | %endif | 871 | %endif |
| @@ -796,15 +891,34 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 19 unmodified lines |
| 796 | %xdefine %1 mangle(private_prefix %+ _ %+ %1) | 891 | %xdefine %1 mangle(private_prefix %+ _ %+ %1) |
| 797 | %if FORMAT_ELF | 892 | %if FORMAT_ELF |
| 798 | global %1:data hidden | 893 | global %1:data hidden |
| | 894 | %elif FORMAT_MACHO && HAVE_PRIVATE_EXTERN |
| | 895 | global %1:private_extern |
| 799 | %else | 896 | %else |
| 800 | global %1 | 897 | global %1 |
| 801 | %endif | 898 | %endif |
| 802 | %1: %2 | 899 | %1: %2 |
| 803 | %endmacro | 900 | %endmacro |
| 804 | | 901 | |
| 805 | ; This is needed for ELF, otherwise the GNU linker assumes the stack is executable by default. | | |
| 806 | %if FORMAT_ELF | 902 | %if FORMAT_ELF |
| | 903 | ; The GNU linker assumes the stack is executable by default. |
| 807 | [SECTION .note.GNU-stack noalloc noexec nowrite progbits] | 904 | [SECTION .note.GNU-stack noalloc noexec nowrite progbits] |
| | 905 | |
| | 906 | %ifdef __NASM_VERSION_ID__ |
| | 907 | %if __NASM_VERSION_ID__ >= 0x020e0300 ; 2.14.03 |
| | 908 | %if ARCH_X86_64 |
| | 909 | ; Control-flow Enforcement Technology (CET) properties. |
| | 910 | [SECTION .note.gnu.property alloc noexec nowrite note align=gprsize] |
| | 911 | dd 0x00000004 ; n_namesz |
| | 912 | dd gprsize + 8 ; n_descsz |
| | 913 | dd 0x00000005 ; n_type = NT_GNU_PROPERTY_TYPE_0 |
| | 914 | db "GNU",0 ; n_name |
| | 915 | dd 0xc0000002 ; pr_type = GNU_PROPERTY_X86_FEATURE_1_AND |
| | 916 | dd 0x00000004 ; pr_datasz |
| | 917 | dd 0x00000002 ; pr_data = GNU_PROPERTY_X86_FEATURE_1_SHSTK |
| | 918 | dd 0x00000000 ; pr_padding |
| | 919 | %endif |
| | 920 | %endif |
| | 921 | %endif |
| 808 | %endif | 922 | %endif |
| 809 | | 923 | |
| 810 | ; Tell debuggers how large the function was. | 924 | ; Tell debuggers how large the function was. |
| @@ -828,32 +942,34 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 17 unmodified lines |
| 828 | ; cpuflags | 942 | ; cpuflags |
| 829 | | 943 | |
| 830 | %assign cpuflags_mmx (1<<0) | 944 | %assign cpuflags_mmx (1<<0) |
| 831 | %assign cpuflags_mmx2 (1<<1) | cpuflags_mmx | 945 | %assign cpuflags_mmx2 (1<<1) | cpuflags_mmx |
| 832 | %assign cpuflags_3dnow (1<<2) | cpuflags_mmx | 946 | %assign cpuflags_3dnow (1<<2) | cpuflags_mmx |
| 833 | %assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow | 947 | %assign cpuflags_3dnowext (1<<3) | cpuflags_3dnow |
| 834 | %assign cpuflags_sse (1<<4) | cpuflags_mmx2 | 948 | %assign cpuflags_sse (1<<4) | cpuflags_mmx2 |
| 835 | %assign cpuflags_sse2 (1<<5) | cpuflags_sse | 949 | %assign cpuflags_sse2 (1<<5) | cpuflags_sse |
| 836 | %assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 | 950 | %assign cpuflags_sse2slow (1<<6) | cpuflags_sse2 |
| 837 | %assign cpuflags_lzcnt (1<<7) | cpuflags_sse2 | 951 | %assign cpuflags_lzcnt (1<<7) | cpuflags_sse2 |
| 838 | %assign cpuflags_sse3 (1<<8) | cpuflags_sse2 | 952 | %assign cpuflags_sse3 (1<<8) | cpuflags_sse2 |
| 839 | %assign cpuflags_ssse3 (1<<9) | cpuflags_sse3 | 953 | %assign cpuflags_ssse3 (1<<9) | cpuflags_sse3 |
| 840 | %assign cpuflags_sse4 (1<<10)| cpuflags_ssse3 | 954 | %assign cpuflags_sse4 (1<<10) | cpuflags_ssse3 |
| 841 | %assign cpuflags_sse42 (1<<11)| cpuflags_sse4 | 955 | %assign cpuflags_sse42 (1<<11) | cpuflags_sse4 |
| 842 | %assign cpuflags_aesni (1<<12)| cpuflags_sse42 | 956 | %assign cpuflags_aesni (1<<12) | cpuflags_sse42 |
| 843 | %assign cpuflags_avx (1<<13)| cpuflags_sse42 | 957 | %assign cpuflags_clmul (1<<13) | cpuflags_sse42 |
| 844 | %assign cpuflags_xop (1<<14)| cpuflags_avx | 958 | %assign cpuflags_gfni (1<<14) | cpuflags_aesni|cpuflags_clmul |
| 845 | %assign cpuflags_fma4 (1<<15)| cpuflags_avx | 959 | %assign cpuflags_avx (1<<15) | cpuflags_sse42 |
| 846 | %assign cpuflags_fma3 (1<<16)| cpuflags_avx | 960 | %assign cpuflags_xop (1<<16) | cpuflags_avx |
| 847 | %assign cpuflags_bmi1 (1<<17)| cpuflags_avx|cpuflags_lzcnt | 961 | %assign cpuflags_fma4 (1<<17) | cpuflags_avx |
| 848 | %assign cpuflags_bmi2 (1<<18)| cpuflags_bmi1 | 962 | %assign cpuflags_fma3 (1<<18) | cpuflags_avx |
| 849 | %assign cpuflags_avx2 (1<<19)| cpuflags_fma3|cpuflags_bmi2 | 963 | %assign cpuflags_bmi1 (1<<19) | cpuflags_avx|cpuflags_lzcnt |
| 850 | %assign cpuflags_avx512 (1<<20)| cpuflags_avx2 ; F, CD, BW, DQ, VL | 964 | %assign cpuflags_bmi2 (1<<20) | cpuflags_bmi1 |
| 851 | %assign cpuflags_avx512icl (1<<25)| cpuflags_avx512 | 965 | %assign cpuflags_avx2 (1<<21) | cpuflags_fma3|cpuflags_bmi2 |
| 852 | | 966 | %assign cpuflags_avx512 (1<<22) | cpuflags_avx2 ; F, CD, BW, DQ, VL |
| 853 | %assign cpuflags_cache32 (1<<21) | 967 | %assign cpuflags_avx512icl (1<<23) | cpuflags_avx512|cpuflags_gfni ; VNNI, IFMA, VBMI, VBMI2, VPOPCNTDQ, BITALG, VAES, VPCLMULQDQ |
| 854 | %assign cpuflags_cache64 (1<<22) | 968 | |
| 855 | %assign cpuflags_aligned (1<<23) ; not a cpu feature, but a function variant | 969 | %assign cpuflags_cache32 (1<<24) |
| 856 | %assign cpuflags_atom (1<<24) | 970 | %assign cpuflags_cache64 (1<<25) |
| | 971 | %assign cpuflags_aligned (1<<26) ; not a cpu feature, but a function variant |
| | 972 | %assign cpuflags_atom (1<<27) |
| 857 | | 973 | |
| 858 | ; Returns a boolean value expressing whether or not the specified cpuflag is enabled. | 974 | ; Returns a boolean value expressing whether or not the specified cpuflag is enabled. |
| 859 | %define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1) | 975 | %define cpuflag(x) (((((cpuflags & (cpuflags_ %+ x)) ^ (cpuflags_ %+ x)) - 1) >> 31) & 1) |
| @@ -895,9 +1011,17 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 35 unmodified lines |
| 895 | %endif | 1011 | %endif |
| 896 | | 1012 | |
| 897 | %if ARCH_X86_64 || cpuflag(sse2) | 1013 | %if ARCH_X86_64 || cpuflag(sse2) |
| 898 | CPUNOP amdnop | 1014 | %ifdef __NASM_VERSION_ID__ |
| | 1015 | ALIGNMODE p6 |
| | 1016 | %else |
| | 1017 | CPU amdnop |
| | 1018 | %endif |
| 899 | %else | 1019 | %else |
| 900 | CPUNOP basicnop | 1020 | %ifdef __NASM_VERSION_ID__ |
| | 1021 | ALIGNMODE nop |
| | 1022 | %else |
| | 1023 | CPU basicnop |
| | 1024 | %endif |
| 901 | %endif | 1025 | %endif |
| 902 | %endmacro | 1026 | %endmacro |
| 903 | | 1027 | |
| @@ -971,7 +1095,7 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 67 unmodified lines |
| 971 | %endmacro | 1095 | %endmacro |
| 972 | | 1096 | |
| 973 | %macro INIT_XMM 0-1+ | 1097 | %macro INIT_XMM 0-1+ |
| 974 | %assign avx_enabled 0 | 1098 | %assign avx_enabled FORCE_VEX_ENCODING |
| 975 | %define RESET_MM_PERMUTATION INIT_XMM %1 | 1099 | %define RESET_MM_PERMUTATION INIT_XMM %1 |
| 976 | %define mmsize 16 | 1100 | %define mmsize 16 |
| 977 | %define mova movdqa | 1101 | %define mova movdqa |
| @@ -983,6 +1107,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 5 unmodified lines |
| 983 | %if WIN64 | 1107 | %if WIN64 |
| 984 | AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers | 1108 | AVX512_MM_PERMUTATION 6 ; Swap callee-saved registers with volatile registers |
| 985 | %endif | 1109 | %endif |
| | 1110 | %xdefine bcstw 1to8 |
| | 1111 | %xdefine bcstd 1to4 |
| | 1112 | %xdefine bcstq 1to2 |
| 986 | %endmacro | 1113 | %endmacro |
| 987 | | 1114 | |
| 988 | %macro INIT_YMM 0-1+ | 1115 | %macro INIT_YMM 0-1+ |
| @@ -996,6 +1123,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 7 unmodified lines |
| 996 | INIT_CPUFLAGS %1 | 1123 | INIT_CPUFLAGS %1 |
| 997 | DEFINE_MMREGS ymm | 1124 | DEFINE_MMREGS ymm |
| 998 | AVX512_MM_PERMUTATION | 1125 | AVX512_MM_PERMUTATION |
| | 1126 | %xdefine bcstw 1to16 |
| | 1127 | %xdefine bcstd 1to8 |
| | 1128 | %xdefine bcstq 1to4 |
| 999 | %endmacro | 1129 | %endmacro |
| 1000 | | 1130 | |
| 1001 | %macro INIT_ZMM 0-1+ | 1131 | %macro INIT_ZMM 0-1+ |
| @@ -1009,6 +1139,9 @@ BRANCH_INSTR jz, je, jnz, jne, jl, jle, jnl, jnle, jg, jge, jng, jnge, ja, jae, · 7 unmodified lines |
| 1009 | INIT_CPUFLAGS %1 | 1139 | INIT_CPUFLAGS %1 |
| 1010 | DEFINE_MMREGS zmm | 1140 | DEFINE_MMREGS zmm |
| 1011 | AVX512_MM_PERMUTATION | 1141 | AVX512_MM_PERMUTATION |
| | 1142 | %xdefine bcstw 1to32 |
| | 1143 | %xdefine bcstd 1to16 |
| | 1144 | %xdefine bcstq 1to8 |
| 1012 | %endmacro | 1145 | %endmacro |
| 1013 | | 1146 | |
| 1014 | INIT_XMM | 1147 | INIT_XMM |
| @@ -1106,19 +1239,32 @@ INIT_XMM · 91 unmodified lines |
| 1106 | %endif | 1239 | %endif |
| 1107 | %assign %%i 0 | 1240 | %assign %%i 0 |
| 1108 | %rep num_mmregs | 1241 | %rep num_mmregs |
| 1109 | CAT_XDEFINE %%f, %%i, m %+ %%i | 1242 | %xdefine %%tmp m %+ %%i |
| | 1243 | CAT_XDEFINE %%f, %%i, regnumof %+ %%tmp |
| 1110 | %assign %%i %%i+1 | 1244 | %assign %%i %%i+1 |
| 1111 | %endrep | 1245 | %endrep |
| 1112 | %endmacro | 1246 | %endmacro |
| 1113 | | 1247 | |
| 1114 | %macro LOAD_MM_PERMUTATION 1 ; name to load from | 1248 | %macro LOAD_MM_PERMUTATION 0-1 ; name to load from |
| 1115 | %ifdef %1_m0 | 1249 | %if %0 |
| | 1250 | %xdefine %%f %1_m |
| | 1251 | %else |
| | 1252 | %xdefine %%f current_function %+ _m |
| | 1253 | %endif |
| | 1254 | %xdefine %%tmp %%f %+ 0 |
| | 1255 | %ifnum %%tmp |
| | 1256 | DEFINE_MMREGS mmtype |
| 1116 | %assign %%i 0 | 1257 | %assign %%i 0 |
| 1117 | %rep num_mmregs | 1258 | %rep num_mmregs |
| 1118 | CAT_XDEFINE m, %%i, %1_m %+ %%i | 1259 | %xdefine %%tmp %%f %+ %%i |
| 1119 | CAT_XDEFINE nn, m %+ %%i, %%i | 1260 | CAT_XDEFINE %%m, %%i, m %+ %%tmp |
| 1120 | %assign %%i %%i+1 | 1261 | %assign %%i %%i+1 |
| 1121 | %endrep | 1262 | %endrep |
| | 1263 | %rep num_mmregs |
| | 1264 | %assign %%i %%i-1 |
| | 1265 | CAT_XDEFINE m, %%i, %%m %+ %%i |
| | 1266 | CAT_XDEFINE nn, m %+ %%i, %%i |
| | 1267 | %endrep |
| 1122 | %endif | 1268 | %endif |
| 1123 | %endmacro | 1269 | %endmacro |
| 1124 | | 1270 | |
| @@ -1224,8 +1370,22 @@ INIT_XMM · 99 unmodified lines |
| 1224 | %ifdef cpuname | 1370 | %ifdef cpuname |
| 1225 | %if notcpuflag(%2) | 1371 | %if notcpuflag(%2) |
| 1226 | %error use of ``%1'' %2 instruction in cpuname function: current_function | 1372 | %error use of ``%1'' %2 instruction in cpuname function: current_function |
| 1227 | %elif cpuflags_%2 < cpuflags_sse && notcpuflag(sse2) && __sizeofreg > 8 | 1373 | %elif %3 == 0 && __sizeofreg == 16 && notcpuflag(sse2) |
| 1228 | %error use of ``%1'' sse2 instruction in cpuname function: current_function | 1374 | %error use of ``%1'' sse2 instruction in cpuname function: current_function |
| | 1375 | %elif %3 == 0 && __sizeofreg == 32 && notcpuflag(avx2) |
| | 1376 | %error use of ``%1'' avx2 instruction in cpuname function: current_function |
| | 1377 | %elif __sizeofreg == 16 && notcpuflag(sse) |
| | 1378 | %error use of ``%1'' sse instruction in cpuname function: current_function |
| | 1379 | %elif __sizeofreg == 32 && notcpuflag(avx) |
| | 1380 | %error use of ``%1'' avx instruction in cpuname function: current_function |
| | 1381 | %elif __sizeofreg == 64 && notcpuflag(avx512) |
| | 1382 | %error use of ``%1'' avx512 instruction in cpuname function: current_function |
| | 1383 | %elifidn %1, pextrw ; special case because the base instruction is mmx2, |
| | 1384 | %ifnid %6 ; but sse4 is required for memory operands |
| | 1385 | %if notcpuflag(sse4) |
| | 1386 | %error use of ``%1'' sse4 instruction in cpuname function: current_function |
| | 1387 | %endif |
| | 1388 | %endif |
| 1229 | %endif | 1389 | %endif |
| 1230 | %endif | 1390 | %endif |
| 1231 | %endif | 1391 | %endif |
| @@ -1267,11 +1427,79 @@ INIT_XMM · 35 unmodified lines |
| 1267 | %1 %6, __src2 | 1427 | %1 %6, __src2 |
| 1268 | %endif | 1428 | %endif |
| 1269 | %elif %0 >= 9 | 1429 | %elif %0 >= 9 |
| 1270 | __instr %6, %7, %8, %9 | 1430 | %if avx_enabled && __sizeofreg >= 16 && %4 == 1 |
| | 1431 | %ifnnum regnumof%7 |
| | 1432 | %if %3 |
| | 1433 | vmovaps %6, %7 |
| | 1434 | %else |
| | 1435 | vmovdqa %6, %7 |
| | 1436 | %endif |
| | 1437 | __instr %6, %6, %8, %9 |
| | 1438 | %else |
| | 1439 | __instr %6, %7, %8, %9 |
| | 1440 | %endif |
| | 1441 | %else |
| | 1442 | __instr %6, %7, %8, %9 |
| | 1443 | %endif |
| 1271 | %elif %0 == 8 | 1444 | %elif %0 == 8 |
| 1272 | __instr %6, %7, %8 | 1445 | %if avx_enabled && __sizeofreg >= 16 && %4 == 0 |
| | 1446 | %xdefine __src1 %7 |
| | 1447 | %xdefine __src2 %8 |
| | 1448 | %if %5 |
| | 1449 | %ifnum regnumof%7 |
| | 1450 | %ifnum regnumof%8 |
| | 1451 | %if regnumof%7 < 8 && regnumof%8 >= 8 && regnumof%8 < 16 && sizeof%8 <= 32 |
| | 1452 | ; Most VEX-encoded instructions require an additional byte to encode when |
| | 1453 | ; src2 is a high register (e.g. m8..15). If the instruction is commutative |
| | 1454 | ; we can swap src1 and src2 when doing so reduces the instruction length. |
| | 1455 | %xdefine __src1 %8 |
| | 1456 | %xdefine __src2 %7 |
| | 1457 | %endif |
| | 1458 | %endif |
| | 1459 | %elifnum regnumof%8 ; put memory operands in src2 when possible |
| | 1460 | %xdefine __src1 %8 |
| | 1461 | %xdefine __src2 %7 |
| | 1462 | %else |
| | 1463 | %assign __emulate_avx 1 |
| | 1464 | %endif |
| | 1465 | %elifnnum regnumof%7 |
| | 1466 | ; EVEX allows imm8 shift instructions to be used with memory operands, |
| | 1467 | ; but VEX does not. This handles those special cases. |
| | 1468 | %ifnnum %8 |
| | 1469 | %assign __emulate_avx 1 |
| | 1470 | %elif notcpuflag(avx512) |
| | 1471 | %assign __emulate_avx 1 |
| | 1472 | %endif |
| | 1473 | %endif |
| | 1474 | %if __emulate_avx ; a separate load is required |
| | 1475 | %if %3 |
| | 1476 | vmovaps %6, %7 |
| | 1477 | %else |
| | 1478 | vmovdqa %6, %7 |
| | 1479 | %endif |
| | 1480 | __instr %6, %6, %8 |
| | 1481 | %else |
| | 1482 | __instr %6, __src1, __src2 |
| | 1483 | %endif |
| | 1484 | %else |
| | 1485 | __instr %6, %7, %8 |
| | 1486 | %endif |
| 1273 | %elif %0 == 7 | 1487 | %elif %0 == 7 |
| 1274 | __instr %6, %7 | 1488 | %if avx_enabled && __sizeofreg >= 16 && %5 |
| | 1489 | %xdefine __src1 %6 |
| | 1490 | %xdefine __src2 %7 |
| | 1491 | %ifnum regnumof%6 |
| | 1492 | %ifnum regnumof%7 |
| | 1493 | %if regnumof%6 < 8 && regnumof%7 >= 8 && regnumof%7 < 16 && sizeof%7 <= 32 |
| | 1494 | %xdefine __src1 %7 |
| | 1495 | %xdefine __src2 %6 |
| | 1496 | %endif |
| | 1497 | %endif |
| | 1498 | %endif |
| | 1499 | __instr %6, __src1, __src2 |
| | 1500 | %else |
| | 1501 | __instr %6, %7 |
| | 1502 | %endif |
| 1275 | %else | 1503 | %else |
| 1276 | __instr %6 | 1504 | __instr %6 |
| 1277 | %endif | 1505 | %endif |
| @@ -1318,8 +1546,8 @@ AVX_INSTR andpd, sse2, 1, 0, 1 · 40 unmodified lines |
| 1318 | AVX_INSTR andps, sse, 1, 0, 1 | 1546 | AVX_INSTR andps, sse, 1, 0, 1 |
| 1319 | AVX_INSTR blendpd, sse4, 1, 1, 0 | 1547 | AVX_INSTR blendpd, sse4, 1, 1, 0 |
| 1320 | AVX_INSTR blendps, sse4, 1, 1, 0 | 1548 | AVX_INSTR blendps, sse4, 1, 1, 0 |
| 1321 | AVX_INSTR blendvpd, sse4 ; can't be emulated | 1549 | AVX_INSTR blendvpd, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding |
| 1322 | AVX_INSTR blendvps, sse4 ; can't be emulated | 1550 | AVX_INSTR blendvps, sse4, 1, 1, 0 ; last operand must be xmm0 with legacy encoding |
| 1323 | AVX_INSTR cmpeqpd, sse2, 1, 0, 1 | 1551 | AVX_INSTR cmpeqpd, sse2, 1, 0, 1 |
| 1324 | AVX_INSTR cmpeqps, sse, 1, 0, 1 | 1552 | AVX_INSTR cmpeqps, sse, 1, 0, 1 |
| 1325 | AVX_INSTR cmpeqsd, sse2, 1, 0, 0 | 1553 | AVX_INSTR cmpeqsd, sse2, 1, 0, 0 |
| @@ -1356,38 +1584,41 @@ AVX_INSTR cmpunordpd, sse2, 1, 0, 1 · 30 unmodified lines |
| 1356 | AVX_INSTR cmpunordps, sse, 1, 0, 1 | 1584 | AVX_INSTR cmpunordps, sse, 1, 0, 1 |
| 1357 | AVX_INSTR cmpunordsd, sse2, 1, 0, 0 | 1585 | AVX_INSTR cmpunordsd, sse2, 1, 0, 0 |
| 1358 | AVX_INSTR cmpunordss, sse, 1, 0, 0 | 1586 | AVX_INSTR cmpunordss, sse, 1, 0, 0 |
| 1359 | AVX_INSTR comisd, sse2 | 1587 | AVX_INSTR comisd, sse2, 1 |
| 1360 | AVX_INSTR comiss, sse | 1588 | AVX_INSTR comiss, sse, 1 |
| 1361 | AVX_INSTR cvtdq2pd, sse2 | 1589 | AVX_INSTR cvtdq2pd, sse2, 1 |
| 1362 | AVX_INSTR cvtdq2ps, sse2 | 1590 | AVX_INSTR cvtdq2ps, sse2, 1 |
| 1363 | AVX_INSTR cvtpd2dq, sse2 | 1591 | AVX_INSTR cvtpd2dq, sse2, 1 |
| 1364 | AVX_INSTR cvtpd2ps, sse2 | 1592 | AVX_INSTR cvtpd2ps, sse2, 1 |
| 1365 | AVX_INSTR cvtps2dq, sse2 | 1593 | AVX_INSTR cvtps2dq, sse2, 1 |
| 1366 | AVX_INSTR cvtps2pd, sse2 | 1594 | AVX_INSTR cvtps2pd, sse2, 1 |
| 1367 | AVX_INSTR cvtsd2si, sse2 | 1595 | AVX_INSTR cvtsd2si, sse2, 1 |
| 1368 | AVX_INSTR cvtsd2ss, sse2, 1, 0, 0 | 1596 | AVX_INSTR cvtsd2ss, sse2, 1, 0, 0 |
| 1369 | AVX_INSTR cvtsi2sd, sse2, 1, 0, 0 | 1597 | AVX_INSTR cvtsi2sd, sse2, 1, 0, 0 |
| 1370 | AVX_INSTR cvtsi2ss, sse, 1, 0, 0 | 1598 | AVX_INSTR cvtsi2ss, sse, 1, 0, 0 |
| 1371 | AVX_INSTR cvtss2sd, sse2, 1, 0, 0 | 1599 | AVX_INSTR cvtss2sd, sse2, 1, 0, 0 |
| 1372 | AVX_INSTR cvtss2si, sse | 1600 | AVX_INSTR cvtss2si, sse, 1 |
| 1373 | AVX_INSTR cvttpd2dq, sse2 | 1601 | AVX_INSTR cvttpd2dq, sse2, 1 |
| 1374 | AVX_INSTR cvttps2dq, sse2 | 1602 | AVX_INSTR cvttps2dq, sse2, 1 |
| 1375 | AVX_INSTR cvttsd2si, sse2 | 1603 | AVX_INSTR cvttsd2si, sse2, 1 |
| 1376 | AVX_INSTR cvttss2si, sse | 1604 | AVX_INSTR cvttss2si, sse, 1 |
| 1377 | AVX_INSTR divpd, sse2, 1, 0, 0 | 1605 | AVX_INSTR divpd, sse2, 1, 0, 0 |
| 1378 | AVX_INSTR divps, sse, 1, 0, 0 | 1606 | AVX_INSTR divps, sse, 1, 0, 0 |
| 1379 | AVX_INSTR divsd, sse2, 1, 0, 0 | 1607 | AVX_INSTR divsd, sse2, 1, 0, 0 |
| 1380 | AVX_INSTR divss, sse, 1, 0, 0 | 1608 | AVX_INSTR divss, sse, 1, 0, 0 |
| 1381 | AVX_INSTR dppd, sse4, 1, 1, 0 | 1609 | AVX_INSTR dppd, sse4, 1, 1, 0 |
| 1382 | AVX_INSTR dpps, sse4, 1, 1, 0 | 1610 | AVX_INSTR dpps, sse4, 1, 1, 0 |
| 1383 | AVX_INSTR extractps, sse4 | 1611 | AVX_INSTR extractps, sse4, 1 |
| | 1612 | AVX_INSTR gf2p8affineinvqb, gfni, 0, 1, 0 |
| | 1613 | AVX_INSTR gf2p8affineqb, gfni, 0, 1, 0 |
| | 1614 | AVX_INSTR gf2p8mulb, gfni, 0, 0, 0 |
| 1384 | AVX_INSTR haddpd, sse3, 1, 0, 0 | 1615 | AVX_INSTR haddpd, sse3, 1, 0, 0 |
| 1385 | AVX_INSTR haddps, sse3, 1, 0, 0 | 1616 | AVX_INSTR haddps, sse3, 1, 0, 0 |
| 1386 | AVX_INSTR hsubpd, sse3, 1, 0, 0 | 1617 | AVX_INSTR hsubpd, sse3, 1, 0, 0 |
| 1387 | AVX_INSTR hsubps, sse3, 1, 0, 0 | 1618 | AVX_INSTR hsubps, sse3, 1, 0, 0 |
| 1388 | AVX_INSTR insertps, sse4, 1, 1, 0 | 1619 | AVX_INSTR insertps, sse4, 1, 1, 0 |
| 1389 | AVX_INSTR lddqu, sse3 | 1620 | AVX_INSTR lddqu, sse3 |
| 1390 | AVX_INSTR ldmxcsr, sse | 1621 | AVX_INSTR ldmxcsr, sse, 1 |
| 1391 | AVX_INSTR maskmovdqu, sse2 | 1622 | AVX_INSTR maskmovdqu, sse2 |
| 1392 | AVX_INSTR maxpd, sse2, 1, 0, 1 | 1623 | AVX_INSTR maxpd, sse2, 1, 0, 1 |
| 1393 | AVX_INSTR maxps, sse, 1, 0, 1 | 1624 | AVX_INSTR maxps, sse, 1, 0, 1 |
| @@ -1397,10 +1628,10 @@ AVX_INSTR minpd, sse2, 1, 0, 1 · 3 unmodified lines |
| 1397 | AVX_INSTR minps, sse, 1, 0, 1 | 1628 | AVX_INSTR minps, sse, 1, 0, 1 |
| 1398 | AVX_INSTR minsd, sse2, 1, 0, 0 | 1629 | AVX_INSTR minsd, sse2, 1, 0, 0 |
| 1399 | AVX_INSTR minss, sse, 1, 0, 0 | 1630 | AVX_INSTR minss, sse, 1, 0, 0 |
| 1400 | AVX_INSTR movapd, sse2 | 1631 | AVX_INSTR movapd, sse2, 1 |
| 1401 | AVX_INSTR movaps, sse | 1632 | AVX_INSTR movaps, sse, 1 |
| 1402 | AVX_INSTR movd, mmx | 1633 | AVX_INSTR movd, mmx |
| 1403 | AVX_INSTR movddup, sse3 | 1634 | AVX_INSTR movddup, sse3, 1 |
| 1404 | AVX_INSTR movdqa, sse2 | 1635 | AVX_INSTR movdqa, sse2 |
| 1405 | AVX_INSTR movdqu, sse2 | 1636 | AVX_INSTR movdqu, sse2 |
| 1406 | AVX_INSTR movhlps, sse, 1, 0, 0 | 1637 | AVX_INSTR movhlps, sse, 1, 0, 0 |
| @@ -1409,19 +1640,19 @@ AVX_INSTR movhps, sse, 1, 0, 0 · 2 unmodified lines |
| 1409 | AVX_INSTR movlhps, sse, 1, 0, 0 | 1640 | AVX_INSTR movlhps, sse, 1, 0, 0 |
| 1410 | AVX_INSTR movlpd, sse2, 1, 0, 0 | 1641 | AVX_INSTR movlpd, sse2, 1, 0, 0 |
| 1411 | AVX_INSTR movlps, sse, 1, 0, 0 | 1642 | AVX_INSTR movlps, sse, 1, 0, 0 |
| 1412 | AVX_INSTR movmskpd, sse2 | 1643 | AVX_INSTR movmskpd, sse2, 1 |
| 1413 | AVX_INSTR movmskps, sse | 1644 | AVX_INSTR movmskps, sse, 1 |
| 1414 | AVX_INSTR movntdq, sse2 | 1645 | AVX_INSTR movntdq, sse2 |
| 1415 | AVX_INSTR movntdqa, sse4 | 1646 | AVX_INSTR movntdqa, sse4 |
| 1416 | AVX_INSTR movntpd, sse2 | 1647 | AVX_INSTR movntpd, sse2, 1 |
| 1417 | AVX_INSTR movntps, sse | 1648 | AVX_INSTR movntps, sse, 1 |
| 1418 | AVX_INSTR movq, mmx | 1649 | AVX_INSTR movq, mmx |
| 1419 | AVX_INSTR movsd, sse2, 1, 0, 0 | 1650 | AVX_INSTR movsd, sse2, 1, 0, 0 |
| 1420 | AVX_INSTR movshdup, sse3 | 1651 | AVX_INSTR movshdup, sse3, 1 |
| 1421 | AVX_INSTR movsldup, sse3 | 1652 | AVX_INSTR movsldup, sse3, 1 |
| 1422 | AVX_INSTR movss, sse, 1, 0, 0 | 1653 | AVX_INSTR movss, sse, 1, 0, 0 |
| 1423 | AVX_INSTR movupd, sse2 | 1654 | AVX_INSTR movupd, sse2, 1 |
| 1424 | AVX_INSTR movups, sse | 1655 | AVX_INSTR movups, sse, 1 |
| 1425 | AVX_INSTR mpsadbw, sse4, 0, 1, 0 | 1656 | AVX_INSTR mpsadbw, sse4, 0, 1, 0 |
| 1426 | AVX_INSTR mulpd, sse2, 1, 0, 1 | 1657 | AVX_INSTR mulpd, sse2, 1, 0, 1 |
| 1427 | AVX_INSTR mulps, sse, 1, 0, 1 | 1658 | AVX_INSTR mulps, sse, 1, 0, 1 |
| @@ -1432,90 +1663,90 @@ AVX_INSTR orps, sse, 1, 0, 1 · 4 unmodified lines |
| 1432 | AVX_INSTR pabsb, ssse3 | 1663 | AVX_INSTR pabsb, ssse3 |
| 1433 | AVX_INSTR pabsd, ssse3 | 1664 | AVX_INSTR pabsd, ssse3 |
| 1434 | AVX_INSTR pabsw, ssse3 | 1665 | AVX_INSTR pabsw, ssse3 |
| 1435 | AVX_INSTR packsswb, mmx, 0, 0, 0 | | |
| 1436 | AVX_INSTR packssdw, mmx, 0, 0, 0 | 1666 | AVX_INSTR packssdw, mmx, 0, 0, 0 |
| 1437 | AVX_INSTR packuswb, mmx, 0, 0, 0 | 1667 | AVX_INSTR packsswb, mmx, 0, 0, 0 |
| 1438 | AVX_INSTR packusdw, sse4, 0, 0, 0 | 1668 | AVX_INSTR packusdw, sse4, 0, 0, 0 |
| | 1669 | AVX_INSTR packuswb, mmx, 0, 0, 0 |
| 1439 | AVX_INSTR paddb, mmx, 0, 0, 1 | 1670 | AVX_INSTR paddb, mmx, 0, 0, 1 |
| 1440 | AVX_INSTR paddw, mmx, 0, 0, 1 | | |
| 1441 | AVX_INSTR paddd, mmx, 0, 0, 1 | 1671 | AVX_INSTR paddd, mmx, 0, 0, 1 |
| 1442 | AVX_INSTR paddq, sse2, 0, 0, 1 | 1672 | AVX_INSTR paddq, sse2, 0, 0, 1 |
| 1443 | AVX_INSTR paddsb, mmx, 0, 0, 1 | 1673 | AVX_INSTR paddsb, mmx, 0, 0, 1 |
| 1444 | AVX_INSTR paddsw, mmx, 0, 0, 1 | 1674 | AVX_INSTR paddsw, mmx, 0, 0, 1 |
| 1445 | AVX_INSTR paddusb, mmx, 0, 0, 1 | 1675 | AVX_INSTR paddusb, mmx, 0, 0, 1 |
| 1446 | AVX_INSTR paddusw, mmx, 0, 0, 1 | 1676 | AVX_INSTR paddusw, mmx, 0, 0, 1 |
| | 1677 | AVX_INSTR paddw, mmx, 0, 0, 1 |
| 1447 | AVX_INSTR palignr, ssse3, 0, 1, 0 | 1678 | AVX_INSTR palignr, ssse3, 0, 1, 0 |
| 1448 | AVX_INSTR pand, mmx, 0, 0, 1 | 1679 | AVX_INSTR pand, mmx, 0, 0, 1 |
| 1449 | AVX_INSTR pandn, mmx, 0, 0, 0 | 1680 | AVX_INSTR pandn, mmx, 0, 0, 0 |
| 1450 | AVX_INSTR pavgb, mmx2, 0, 0, 1 | 1681 | AVX_INSTR pavgb, mmx2, 0, 0, 1 |
| 1451 | AVX_INSTR pavgw, mmx2, 0, 0, 1 | 1682 | AVX_INSTR pavgw, mmx2, 0, 0, 1 |
| 1452 | AVX_INSTR pblendvb, sse4 ; can't be emulated | 1683 | AVX_INSTR pblendvb, sse4, 0, 1, 0 ; last operand must be xmm0 with legacy encoding |
| 1453 | AVX_INSTR pblendw, sse4, 0, 1, 0 | 1684 | AVX_INSTR pblendw, sse4, 0, 1, 0 |
| 1454 | AVX_INSTR pclmulqdq, fnord, 0, 1, 0 | 1685 | AVX_INSTR pclmulhqhqdq, clmul, 0, 0, 0 |
| 1455 | AVX_INSTR pclmulhqhqdq, fnord, 0, 0, 0 | 1686 | AVX_INSTR pclmulhqlqdq, clmul, 0, 0, 0 |
| 1456 | AVX_INSTR pclmulhqlqdq, fnord, 0, 0, 0 | 1687 | AVX_INSTR pclmullqhqdq, clmul, 0, 0, 0 |
| 1457 | AVX_INSTR pclmullqhqdq, fnord, 0, 0, 0 | 1688 | AVX_INSTR pclmullqlqdq, clmul, 0, 0, 0 |
| 1458 | AVX_INSTR pclmullqlqdq, fnord, 0, 0, 0 | 1689 | AVX_INSTR pclmulqdq, clmul, 0, 1, 0 |
| 1459 | AVX_INSTR pcmpestri, sse42 | | |
| 1460 | AVX_INSTR pcmpestrm, sse42 | | |
| 1461 | AVX_INSTR pcmpistri, sse42 | | |
| 1462 | AVX_INSTR pcmpistrm, sse42 | | |
| 1463 | AVX_INSTR pcmpeqb, mmx, 0, 0, 1 | 1690 | AVX_INSTR pcmpeqb, mmx, 0, 0, 1 |
| 1464 | AVX_INSTR pcmpeqw, mmx, 0, 0, 1 | | |
| 1465 | AVX_INSTR pcmpeqd, mmx, 0, 0, 1 | 1691 | AVX_INSTR pcmpeqd, mmx, 0, 0, 1 |
| 1466 | AVX_INSTR pcmpeqq, sse4, 0, 0, 1 | 1692 | AVX_INSTR pcmpeqq, sse4, 0, 0, 1 |
| | 1693 | AVX_INSTR pcmpeqw, mmx, 0, 0, 1 |
| | 1694 | AVX_INSTR pcmpestri, sse42 |
| | 1695 | AVX_INSTR pcmpestrm, sse42 |
| 1467 | AVX_INSTR pcmpgtb, mmx, 0, 0, 0 | 1696 | AVX_INSTR pcmpgtb, mmx, 0, 0, 0 |
| 1468 | AVX_INSTR pcmpgtw, mmx, 0, 0, 0 | | |
| 1469 | AVX_INSTR pcmpgtd, mmx, 0, 0, 0 | 1697 | AVX_INSTR pcmpgtd, mmx, 0, 0, 0 |
| 1470 | AVX_INSTR pcmpgtq, sse42, 0, 0, 0 | 1698 | AVX_INSTR pcmpgtq, sse42, 0, 0, 0 |
| | 1699 | AVX_INSTR pcmpgtw, mmx, 0, 0, 0 |
| | 1700 | AVX_INSTR pcmpistri, sse42 |
| | 1701 | AVX_INSTR pcmpistrm, sse42 |
| 1471 | AVX_INSTR pextrb, sse4 | 1702 | AVX_INSTR pextrb, sse4 |
| 1472 | AVX_INSTR pextrd, sse4 | 1703 | AVX_INSTR pextrd, sse4 |
| 1473 | AVX_INSTR pextrq, sse4 | 1704 | AVX_INSTR pextrq, sse4 |
| 1474 | AVX_INSTR pextrw, mmx2 | 1705 | AVX_INSTR pextrw, mmx2 |
| 1475 | AVX_INSTR phaddw, ssse3, 0, 0, 0 | | |
| 1476 | AVX_INSTR phaddd, ssse3, 0, 0, 0 | 1706 | AVX_INSTR phaddd, ssse3, 0, 0, 0 |
| 1477 | AVX_INSTR phaddsw, ssse3, 0, 0, 0 | 1707 | AVX_INSTR phaddsw, ssse3, 0, 0, 0 |
| | 1708 | AVX_INSTR phaddw, ssse3, 0, 0, 0 |
| 1478 | AVX_INSTR phminposuw, sse4 | 1709 | AVX_INSTR phminposuw, sse4 |
| 1479 | AVX_INSTR phsubw, ssse3, 0, 0, 0 | | |
| 1480 | AVX_INSTR phsubd, ssse3, 0, 0, 0 | 1710 | AVX_INSTR phsubd, ssse3, 0, 0, 0 |
| 1481 | AVX_INSTR phsubsw, ssse3, 0, 0, 0 | 1711 | AVX_INSTR phsubsw, ssse3, 0, 0, 0 |
| | 1712 | AVX_INSTR phsubw, ssse3, 0, 0, 0 |
| 1482 | AVX_INSTR pinsrb, sse4, 0, 1, 0 | 1713 | AVX_INSTR pinsrb, sse4, 0, 1, 0 |
| 1483 | AVX_INSTR pinsrd, sse4, 0, 1, 0 | 1714 | AVX_INSTR pinsrd, sse4, 0, 1, 0 |
| 1484 | AVX_INSTR pinsrq, sse4, 0, 1, 0 | 1715 | AVX_INSTR pinsrq, sse4, 0, 1, 0 |
| 1485 | AVX_INSTR pinsrw, mmx2, 0, 1, 0 | 1716 | AVX_INSTR pinsrw, mmx2, 0, 1, 0 |
| 1486 | AVX_INSTR pmaddwd, mmx, 0, 0, 1 | | |
| 1487 | AVX_INSTR pmaddubsw, ssse3, 0, 0, 0 | 1717 | AVX_INSTR pmaddubsw, ssse3, 0, 0, 0 |
| | 1718 | AVX_INSTR pmaddwd, mmx, 0, 0, 1 |
| 1488 | AVX_INSTR pmaxsb, sse4, 0, 0, 1 | 1719 | AVX_INSTR pmaxsb, sse4, 0, 0, 1 |
| 1489 | AVX_INSTR pmaxsw, mmx2, 0, 0, 1 | | |
| 1490 | AVX_INSTR pmaxsd, sse4, 0, 0, 1 | 1720 | AVX_INSTR pmaxsd, sse4, 0, 0, 1 |
| | 1721 | AVX_INSTR pmaxsw, mmx2, 0, 0, 1 |
| 1491 | AVX_INSTR pmaxub, mmx2, 0, 0, 1 | 1722 | AVX_INSTR pmaxub, mmx2, 0, 0, 1 |
| 1492 | AVX_INSTR pmaxuw, sse4, 0, 0, 1 | | |
| 1493 | AVX_INSTR pmaxud, sse4, 0, 0, 1 | 1723 | AVX_INSTR pmaxud, sse4, 0, 0, 1 |
| | 1724 | AVX_INSTR pmaxuw, sse4, 0, 0, 1 |
| 1494 | AVX_INSTR pminsb, sse4, 0, 0, 1 | 1725 | AVX_INSTR pminsb, sse4, 0, 0, 1 |
| 1495 | AVX_INSTR pminsw, mmx2, 0, 0, 1 | | |
| 1496 | AVX_INSTR pminsd, sse4, 0, 0, 1 | 1726 | AVX_INSTR pminsd, sse4, 0, 0, 1 |
| | 1727 | AVX_INSTR pminsw, mmx2, 0, 0, 1 |
| 1497 | AVX_INSTR pminub, mmx2, 0, 0, 1 | 1728 | AVX_INSTR pminub, mmx2, 0, 0, 1 |
| 1498 | AVX_INSTR pminuw, sse4, 0, 0, 1 | | |
| 1499 | AVX_INSTR pminud, sse4, 0, 0, 1 | 1729 | AVX_INSTR pminud, sse4, 0, 0, 1 |
| | 1730 | AVX_INSTR pminuw, sse4, 0, 0, 1 |
| 1500 | AVX_INSTR pmovmskb, mmx2 | 1731 | AVX_INSTR pmovmskb, mmx2 |
| 1501 | AVX_INSTR pmovsxbw, sse4 | | |
| 1502 | AVX_INSTR pmovsxbd, sse4 | 1732 | AVX_INSTR pmovsxbd, sse4 |
| 1503 | AVX_INSTR pmovsxbq, sse4 | 1733 | AVX_INSTR pmovsxbq, sse4 |
| | 1734 | AVX_INSTR pmovsxbw, sse4 |
| | 1735 | AVX_INSTR pmovsxdq, sse4 |
| 1504 | AVX_INSTR pmovsxwd, sse4 | 1736 | AVX_INSTR pmovsxwd, sse4 |
| 1505 | AVX_INSTR pmovsxwq, sse4 | 1737 | AVX_INSTR pmovsxwq, sse4 |
| 1506 | AVX_INSTR pmovsxdq, sse4 | | |
| 1507 | AVX_INSTR pmovzxbw, sse4 | | |
| 1508 | AVX_INSTR pmovzxbd, sse4 | 1738 | AVX_INSTR pmovzxbd, sse4 |
| 1509 | AVX_INSTR pmovzxbq, sse4 | 1739 | AVX_INSTR pmovzxbq, sse4 |
| | 1740 | AVX_INSTR pmovzxbw, sse4 |
| | 1741 | AVX_INSTR pmovzxdq, sse4 |
| 1510 | AVX_INSTR pmovzxwd, sse4 | 1742 | AVX_INSTR pmovzxwd, sse4 |
| 1511 | AVX_INSTR pmovzxwq, sse4 | 1743 | AVX_INSTR pmovzxwq, sse4 |
| 1512 | AVX_INSTR pmovzxdq, sse4 | | |
| 1513 | AVX_INSTR pmuldq, sse4, 0, 0, 1 | 1744 | AVX_INSTR pmuldq, sse4, 0, 0, 1 |
| 1514 | AVX_INSTR pmulhrsw, ssse3, 0, 0, 1 | 1745 | AVX_INSTR pmulhrsw, ssse3, 0, 0, 1 |
| 1515 | AVX_INSTR pmulhuw, mmx2, 0, 0, 1 | 1746 | AVX_INSTR pmulhuw, mmx2, 0, 0, 1 |
| 1516 | AVX_INSTR pmulhw, mmx, 0, 0, 1 | 1747 | AVX_INSTR pmulhw, mmx, 0, 0, 1 |
| 1517 | AVX_INSTR pmullw, mmx, 0, 0, 1 | | |
| 1518 | AVX_INSTR pmulld, sse4, 0, 0, 1 | 1748 | AVX_INSTR pmulld, sse4, 0, 0, 1 |
| | 1749 | AVX_INSTR pmullw, mmx, 0, 0, 1 |
| 1519 | AVX_INSTR pmuludq, sse2, 0, 0, 1 | 1750 | AVX_INSTR pmuludq, sse2, 0, 0, 1 |
| 1520 | AVX_INSTR por, mmx, 0, 0, 1 | 1751 | AVX_INSTR por, mmx, 0, 0, 1 |
| 1521 | AVX_INSTR psadbw, mmx2, 0, 0, 1 | 1752 | AVX_INSTR psadbw, mmx2, 0, 0, 1 |
| @@ -1524,57 +1755,57 @@ AVX_INSTR pshufd, sse2 · 2 unmodified lines |
| 1524 | AVX_INSTR pshufhw, sse2 | 1755 | AVX_INSTR pshufhw, sse2 |
| 1525 | AVX_INSTR pshuflw, sse2 | 1756 | AVX_INSTR pshuflw, sse2 |
| 1526 | AVX_INSTR psignb, ssse3, 0, 0, 0 | 1757 | AVX_INSTR psignb, ssse3, 0, 0, 0 |
| 1527 | AVX_INSTR psignw, ssse3, 0, 0, 0 | | |
| 1528 | AVX_INSTR psignd, ssse3, 0, 0, 0 | 1758 | AVX_INSTR psignd, ssse3, 0, 0, 0 |
| 1529 | AVX_INSTR psllw, mmx, 0, 0, 0 | 1759 | AVX_INSTR psignw, ssse3, 0, 0, 0 |
| 1530 | AVX_INSTR pslld, mmx, 0, 0, 0 | 1760 | AVX_INSTR pslld, mmx, 0, 0, 0 |
| 1531 | AVX_INSTR psllq, mmx, 0, 0, 0 | | |
| 1532 | AVX_INSTR pslldq, sse2, 0, 0, 0 | 1761 | AVX_INSTR pslldq, sse2, 0, 0, 0 |
| 1533 | AVX_INSTR psraw, mmx, 0, 0, 0 | 1762 | AVX_INSTR psllq, mmx, 0, 0, 0 |
| | 1763 | AVX_INSTR psllw, mmx, 0, 0, 0 |
| 1534 | AVX_INSTR psrad, mmx, 0, 0, 0 | 1764 | AVX_INSTR psrad, mmx, 0, 0, 0 |
| 1535 | AVX_INSTR psrlw, mmx, 0, 0, 0 | 1765 | AVX_INSTR psraw, mmx, 0, 0, 0 |
| 1536 | AVX_INSTR psrld, mmx, 0, 0, 0 | 1766 | AVX_INSTR psrld, mmx, 0, 0, 0 |
| 1537 | AVX_INSTR psrlq, mmx, 0, 0, 0 | | |
| 1538 | AVX_INSTR psrldq, sse2, 0, 0, 0 | 1767 | AVX_INSTR psrldq, sse2, 0, 0, 0 |
| | 1768 | AVX_INSTR psrlq, mmx, 0, 0, 0 |
| | 1769 | AVX_INSTR psrlw, mmx, 0, 0, 0 |
| 1539 | AVX_INSTR psubb, mmx, 0, 0, 0 | 1770 | AVX_INSTR psubb, mmx, 0, 0, 0 |
| 1540 | AVX_INSTR psubw, mmx, 0, 0, 0 | | |
| 1541 | AVX_INSTR psubd, mmx, 0, 0, 0 | 1771 | AVX_INSTR psubd, mmx, 0, 0, 0 |
| 1542 | AVX_INSTR psubq, sse2, 0, 0, 0 | 1772 | AVX_INSTR psubq, sse2, 0, 0, 0 |
| 1543 | AVX_INSTR psubsb, mmx, 0, 0, 0 | 1773 | AVX_INSTR psubsb, mmx, 0, 0, 0 |
| 1544 | AVX_INSTR psubsw, mmx, 0, 0, 0 | 1774 | AVX_INSTR psubsw, mmx, 0, 0, 0 |
| 1545 | AVX_INSTR psubusb, mmx, 0, 0, 0 | 1775 | AVX_INSTR psubusb, mmx, 0, 0, 0 |
| 1546 | AVX_INSTR psubusw, mmx, 0, 0, 0 | 1776 | AVX_INSTR psubusw, mmx, 0, 0, 0 |
| | 1777 | AVX_INSTR psubw, mmx, 0, 0, 0 |
| 1547 | AVX_INSTR ptest, sse4 | 1778 | AVX_INSTR ptest, sse4 |
| 1548 | AVX_INSTR punpckhbw, mmx, 0, 0, 0 | 1779 | AVX_INSTR punpckhbw, mmx, 0, 0, 0 |
| 1549 | AVX_INSTR punpckhwd, mmx, 0, 0, 0 | | |
| 1550 | AVX_INSTR punpckhdq, mmx, 0, 0, 0 | 1780 | AVX_INSTR punpckhdq, mmx, 0, 0, 0 |
| 1551 | AVX_INSTR punpckhqdq, sse2, 0, 0, 0 | 1781 | AVX_INSTR punpckhqdq, sse2, 0, 0, 0 |
| | 1782 | AVX_INSTR punpckhwd, mmx, 0, 0, 0 |
| 1552 | AVX_INSTR punpcklbw, mmx, 0, 0, 0 | 1783 | AVX_INSTR punpcklbw, mmx, 0, 0, 0 |
| 1553 | AVX_INSTR punpcklwd, mmx, 0, 0, 0 | | |
| 1554 | AVX_INSTR punpckldq, mmx, 0, 0, 0 | 1784 | AVX_INSTR punpckldq, mmx, 0, 0, 0 |
| 1555 | AVX_INSTR punpcklqdq, sse2, 0, 0, 0 | 1785 | AVX_INSTR punpcklqdq, sse2, 0, 0, 0 |
| | 1786 | AVX_INSTR punpcklwd, mmx, 0, 0, 0 |
| 1556 | AVX_INSTR pxor, mmx, 0, 0, 1 | 1787 | AVX_INSTR pxor, mmx, 0, 0, 1 |
| 1557 | AVX_INSTR rcpps, sse | 1788 | AVX_INSTR rcpps, sse, 1 |
| 1558 | AVX_INSTR rcpss, sse, 1, 0, 0 | 1789 | AVX_INSTR rcpss, sse, 1, 0, 0 |
| 1559 | AVX_INSTR roundpd, sse4 | 1790 | AVX_INSTR roundpd, sse4, 1 |
| 1560 | AVX_INSTR roundps, sse4 | 1791 | AVX_INSTR roundps, sse4, 1 |
| 1561 | AVX_INSTR roundsd, sse4, 1, 1, 0 | 1792 | AVX_INSTR roundsd, sse4, 1, 1, 0 |
| 1562 | AVX_INSTR roundss, sse4, 1, 1, 0 | 1793 | AVX_INSTR roundss, sse4, 1, 1, 0 |
| 1563 | AVX_INSTR rsqrtps, sse | 1794 | AVX_INSTR rsqrtps, sse, 1 |
| 1564 | AVX_INSTR rsqrtss, sse, 1, 0, 0 | 1795 | AVX_INSTR rsqrtss, sse, 1, 0, 0 |
| 1565 | AVX_INSTR shufpd, sse2, 1, 1, 0 | 1796 | AVX_INSTR shufpd, sse2, 1, 1, 0 |
| 1566 | AVX_INSTR shufps, sse, 1, 1, 0 | 1797 | AVX_INSTR shufps, sse, 1, 1, 0 |
| 1567 | AVX_INSTR sqrtpd, sse2 | 1798 | AVX_INSTR sqrtpd, sse2, 1 |
| 1568 | AVX_INSTR sqrtps, sse | 1799 | AVX_INSTR sqrtps, sse, 1 |
| 1569 | AVX_INSTR sqrtsd, sse2, 1, 0, 0 | 1800 | AVX_INSTR sqrtsd, sse2, 1, 0, 0 |
| 1570 | AVX_INSTR sqrtss, sse, 1, 0, 0 | 1801 | AVX_INSTR sqrtss, sse, 1, 0, 0 |
| 1571 | AVX_INSTR stmxcsr, sse | 1802 | AVX_INSTR stmxcsr, sse, 1 |
| 1572 | AVX_INSTR subpd, sse2, 1, 0, 0 | 1803 | AVX_INSTR subpd, sse2, 1, 0, 0 |
| 1573 | AVX_INSTR subps, sse, 1, 0, 0 | 1804 | AVX_INSTR subps, sse, 1, 0, 0 |
| 1574 | AVX_INSTR subsd, sse2, 1, 0, 0 | 1805 | AVX_INSTR subsd, sse2, 1, 0, 0 |
| 1575 | AVX_INSTR subss, sse, 1, 0, 0 | 1806 | AVX_INSTR subss, sse, 1, 0, 0 |
| 1576 | AVX_INSTR ucomisd, sse2 | 1807 | AVX_INSTR ucomisd, sse2, 1 |
| 1577 | AVX_INSTR ucomiss, sse | 1808 | AVX_INSTR ucomiss, sse, 1 |
| 1578 | AVX_INSTR unpckhpd, sse2, 1, 0, 0 | 1809 | AVX_INSTR unpckhpd, sse2, 1, 0, 0 |
| 1579 | AVX_INSTR unpckhps, sse, 1, 0, 0 | 1810 | AVX_INSTR unpckhps, sse, 1, 0, 0 |
| 1580 | AVX_INSTR unpcklpd, sse2, 1, 0, 0 | 1811 | AVX_INSTR unpcklpd, sse2, 1, 0, 0 |
| @@ -1584,8 +1815,41 @@ AVX_INSTR xorps, sse, 1, 0, 1 · 3 unmodified lines |
| 1584 | | 1815 | |
| 1585 | ; 3DNow instructions, for sharing code between AVX, SSE and 3DN | 1816 | ; 3DNow instructions, for sharing code between AVX, SSE and 3DN |
| 1586 | AVX_INSTR pfadd, 3dnow, 1, 0, 1 | 1817 | AVX_INSTR pfadd, 3dnow, 1, 0, 1 |
| 1587 | AVX_INSTR pfsub, 3dnow, 1, 0, 0 | | |
| 1588 | AVX_INSTR pfmul, 3dnow, 1, 0, 1 | 1818 | AVX_INSTR pfmul, 3dnow, 1, 0, 1 |
| | 1819 | AVX_INSTR pfsub, 3dnow, 1, 0, 0 |
| | 1820 | |
| | 1821 | ;%1 == instruction |
| | 1822 | ;%2 == minimal instruction set |
| | 1823 | %macro GPR_INSTR 2 |
| | 1824 | %macro %1 2-5 fnord, %1, %2 |
| | 1825 | %ifdef cpuname |
| | 1826 | %if notcpuflag(%5) |
| | 1827 | %error use of ``%4'' %5 instruction in cpuname function: current_function |
| | 1828 | %endif |
| | 1829 | %endif |
| | 1830 | %ifidn %3, fnord |
| | 1831 | %4 %1, %2 |
| | 1832 | %else |
| | 1833 | %4 %1, %2, %3 |
| | 1834 | %endif |
| | 1835 | %endmacro |
| | 1836 | %endmacro |
| | 1837 | |
| | 1838 | GPR_INSTR andn, bmi1 |
| | 1839 | GPR_INSTR bextr, bmi1 |
| | 1840 | GPR_INSTR blsi, bmi1 |
| | 1841 | GPR_INSTR blsmsk, bmi1 |
| | 1842 | GPR_INSTR blsr, bmi1 |
| | 1843 | GPR_INSTR bzhi, bmi2 |
| | 1844 | GPR_INSTR crc32, sse42 |
| | 1845 | GPR_INSTR mulx, bmi2 |
| | 1846 | GPR_INSTR pdep, bmi2 |
| | 1847 | GPR_INSTR pext, bmi2 |
| | 1848 | GPR_INSTR popcnt, sse42 |
| | 1849 | GPR_INSTR rorx, bmi2 |
| | 1850 | GPR_INSTR sarx, bmi2 |
| | 1851 | GPR_INSTR shlx, bmi2 |
| | 1852 | GPR_INSTR shrx, bmi2 |
| 1589 | | 1853 | |
| 1590 | ; base-4 constants for shuffles | 1854 | ; base-4 constants for shuffles |
| 1591 | %assign i 0 | 1855 | %assign i 0 |
| @@ -1618,15 +1882,11 @@ AVX_INSTR pfmul, 3dnow, 1, 0, 1 · 26 unmodified lines |
| 1618 | %endmacro | 1882 | %endmacro |
| 1619 | %endmacro | 1883 | %endmacro |
| 1620 | | 1884 | |
| 1621 | FMA_INSTR pmacsww, pmullw, paddw | 1885 | FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation |
| 1622 | FMA_INSTR pmacsdd, pmulld, paddd ; sse4 emulation | 1886 | FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation |
| 1623 | FMA_INSTR pmacsdql, pmuldq, paddq ; sse4 emulation | 1887 | FMA_INSTR pmacsww, pmullw, paddw |
| 1624 | FMA_INSTR pmadcswd, pmaddwd, paddd | 1888 | FMA_INSTR pmadcswd, pmaddwd, paddd |
| 1625 | | 1889 | |
| 1626 | ; tzcnt is equivalent to "rep bsf" and is backwards-compatible with bsf. | | |
| 1627 | ; This lets us use tzcnt without bumping the yasm version requirement yet. | | |
| 1628 | %define tzcnt rep bsf | | |
| 1629 | | | |
| 1630 | ; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax. | 1890 | ; Macros for consolidating FMA3 and FMA4 using 4-operand (dst, src1, src2, src3) syntax. |
| 1631 | ; FMA3 is only possible if dst is the same as one of the src registers. | 1891 | ; FMA3 is only possible if dst is the same as one of the src registers. |
| 1632 | ; Either src2 or src3 can be a memory operand. | 1892 | ; Either src2 or src3 can be a memory operand. |
| @@ -1687,6 +1947,11 @@ FMA4_INSTR fnmsub, pd, ps, sd, ss · 54 unmodified lines |
| 1687 | %assign %%evex_required 1 | 1947 | %assign %%evex_required 1 |
| 1688 | %endif | 1948 | %endif |
| 1689 | %endif | 1949 | %endif |
| | 1950 | %ifnum regnumof%3 |
| | 1951 | %if regnumof%3 >= 16 || sizeof%3 > 32 |
| | 1952 | %assign %%evex_required 1 |
| | 1953 | %endif |
| | 1954 | %endif |
| 1690 | %if %%evex_required | 1955 | %if %%evex_required |
| 1691 | %6 %%args | 1956 | %6 %%args |
| 1692 | %else | 1957 | %else |
| @@ -1711,16 +1976,3 @@ EVEX_INSTR vrcpps, vrcp14ps, 1 ; EVEX versions have higher precision · 18 unmodified lines |
| 1711 | EVEX_INSTR vrcpss, vrcp14ss, 1 | 1976 | EVEX_INSTR vrcpss, vrcp14ss, 1 |
| 1712 | EVEX_INSTR vrsqrtps, vrsqrt14ps, 1 | 1977 | EVEX_INSTR vrsqrtps, vrsqrt14ps, 1 |
| 1713 | EVEX_INSTR vrsqrtss, vrsqrt14ss, 1 | 1978 | EVEX_INSTR vrsqrtss, vrsqrt14ss, 1 |
| 1714 | | | |
| 1715 | ; workaround: vpbroadcastq is broken in x86_32 due to a yasm bug (fixed in 1.3.0) | | |
| 1716 | %ifdef __YASM_VER__ | | |
| 1717 | %if __YASM_VERSION_ID__ < 0x01030000 && ARCH_X86_64 == 0 | | |
| 1718 | %macro vpbroadcastq 2 | | |
| 1719 | %if sizeof%1 == 16 | | |
| 1720 | movddup %1, %2 | | |
| 1721 | %else | | |
| 1722 | vbroadcastsd %1, %2 | | |
| 1723 | %endif | | |
| 1724 | %endmacro | | |
| 1725 | %endif | | |
| 1726 | %endif | | |