/*** The core.internal.atomic module comtains the low-level atomic features available in hardware.* This module may be a routing layer for compiler intrinsics.** Copyright: Copyright Manu Evans 2019.* License:   $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)* Authors:   Sean Kelly, Alex Rønne Petersen, Manu Evans* Source:    $(DRUNTIMESRC core/internal/_atomic.d)*/ module core.internal.atomic; import core.atomic : has128BitCAS, MemoryOrder; version (DigitalMars)version (AArch64){    /* These functions are all stubbed out. They await someone who knows what       they are doing with AArch64 atomics.       TODO AArch64     */    enum IsAtomicLockFree(T) = T.sizeof <= size_t.sizeof * 2;     inout(T) atomicLoad(MemoryOrder order = MemoryOrder.seq, T)(inout(T)* src) pure nothrow @nogc @trusted        if (CanCAS!T)    {        return *src;    }     void atomicStore(MemoryOrder order = MemoryOrder.seq, T)(T* dest, T value) pure nothrow @nogc @trusted        if (CanCAS!T)    {        *dest = value;    }     T atomicFetchAdd(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted        if (is(T : ulong))    {	return *dest + value;    }     T atomicFetchSub(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted        if (is(T : ulong))    {        return atomicFetchAdd(dest, cast(T)-cast(IntOrLong!T)value);    }     T atomicExchange(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted    if (CanCAS!T)    {        size_t storage = void;        return *cast(T*)&storage;    }     alias atomicCompareExchangeWeak = atomicCompareExchangeStrong;     bool atomicCompareExchangeStrong(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted        if (CanCAS!T)    {        if (*dest != *compare)        {            *compare = *dest;            return false;        }        *dest = value;        return true;    }     alias atomicCompareExchangeWeakNoResult = atomicCompareExchangeStrongNoResult;     bool atomicCompareExchangeStrongNoResult(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, const T compare, T value) pure nothrow @nogc @trusted        if (CanCAS!T)    {        if (*dest != compare)            return false;        *dest = value;        return true;    }     void atomicFence(MemoryOrder order = MemoryOrder.seq)() pure nothrow @nogc @trusted    {    }     void atomicSignalFence(MemoryOrder order = MemoryOrder.seq)() pure nothrow @nogc @trusted    {        // no-op, dmd doesn't reorder instructions    }     void pause() pure nothrow @nogc @trusted    {    }}else // X86 and X86_64{    private    {        enum : int        {            AX, BX, CX, DX, DI, SI, R8, R9        }         immutable string[4][8] registerNames = [            [ "AL", "AX", "EAX", "RAX" ],            [ "BL", "BX", "EBX", "RBX" ],            [ "CL", "CX", "ECX", "RCX" ],            [ "DL", "DX", "EDX", "RDX" ],            [ "DIL", "DI", "EDI", "RDI" ],            [ "SIL", "SI", "ESI", "RSI" ],            [ "R8B", "R8W", "R8D", "R8" ],            [ "R9B", "R9W", "R9D", "R9" ],        ];         template RegIndex(T)        {            static if (T.sizeof == 1)                enum RegIndex = 0;            else static if (T.sizeof == 2)                enum RegIndex = 1;            else static if (T.sizeof == 4)                enum RegIndex = 2;            else static if (T.sizeof == 8)                enum RegIndex = 3;            else                static assert(false, "Invalid type");        }         enum SizedReg(int reg, T = size_t) = registerNames[reg][RegIndex!T];    }     enum IsAtomicLockFree(T) = T.sizeof <= size_t.sizeof * 2;     inout(T) atomicLoad(MemoryOrder order = MemoryOrder.seq, T)(inout(T)* src) pure nothrow @nogc @trusted        if (CanCAS!T)    {        static assert(order != MemoryOrder.rel && order != MemoryOrder.acq_rel,                      "invalid MemoryOrder for atomicLoad()");         // We place some storage on the stack,        //  get a pointer to that (which is also stored on the stack)        //  and then store the result of the load into the storage.        // Finally returning it.        // Anything other than this is calling convention specific,        //  and that is very fail heavy.        size_t[2] storage = void;        size_t* resultValuePtr = cast(size_t*)&storage[0];         static if (T.sizeof == size_t.sizeof * 2)        {            version (D_InlineAsm_X86)            {                asm pure nothrow @nogc @trusted                {                    push EBX; // call preserved                    push EDI;                     mov EBX, 0;                    mov ECX, 0;                    mov EAX, 0;                    mov EDX, 0;                     mov EDI, src;                    lock; cmpxchg8b [EDI];                     lea EBX, resultValuePtr;                    mov EBX, [EBX];                    mov [EBX], EAX;                    mov [EBX + size_t.sizeof], EDX;                     pop EDI;                    pop EBX;                }            }            else version (D_InlineAsm_X86_64)            {                asm pure nothrow @nogc @trusted                {                    push RBX; // call preserved                     mov RBX, 0;                    mov RCX, 0;                    mov RAX, 0;                    mov RDX, 0;                     mov R8, src;                    lock; cmpxchg16b [R8];                     lea RBX, resultValuePtr;                    mov RBX, [RBX];                    mov [RBX], RAX;                    mov [RBX + size_t.sizeof], RDX;                     pop RBX;                }            }            else                static assert(0, "Operation not supported");             return *cast(inout(T)*)resultValuePtr;        }        else static if (needsLoadBarrier!order)        {            version (D_InlineAsm_X86)            {            }            else version (D_InlineAsm_X86_64)            {            }            else                static assert(0, "Operation not supported");             enum SrcReg = SizedReg!CX;            enum ZeroReg = SizedReg!(DX, T);            enum ResReg = SizedReg!(AX, T);            enum TemporaryReg = SizedReg!(BX);             mixin (simpleFormat(q{                asm pure nothrow @nogc @trusted                {                    push %3; // call preserved                     mov %1, 0;                    mov %2, 0;                    mov %0, src;                    lock; cmpxchg [%0], %1;                    lea %3, resultValuePtr;                    mov %3, [%3];                    mov [%3], %2;                     pop %3;                }            }, [SrcReg, ZeroReg, ResReg, TemporaryReg]));             return *cast(inout(T)*)resultValuePtr;        }        else            return *src;    }     void atomicStore(MemoryOrder order = MemoryOrder.seq, T)(T* dest, T value) pure nothrow @nogc @trusted        if (CanCAS!T)    {        static assert(order != MemoryOrder.acq && order != MemoryOrder.acq_rel,                      "Invalid MemoryOrder for atomicStore()");         static if (T.sizeof == size_t.sizeof * 2)        {            version (D_InlineAsm_X86)            {                asm pure nothrow @nogc @trusted                {                    push EDI;                    push EBX;                    lea EDI, value;                    mov EBX, [EDI];                    mov ECX, 4[EDI];                    mov EDI, dest;                    mov EAX, [EDI];                    mov EDX, 4[EDI];                L1: lock; cmpxchg8b [EDI];                    jne L1;                    pop EBX;                    pop EDI;                }            }            else version (D_InlineAsm_X86_64)            {                version (Windows)                {                    asm pure nothrow @nogc @trusted                    {                        naked;                        push RBX;                        mov R8, RDX;                        mov RAX, [RDX];                        mov RDX, 8[RDX];                        mov RBX, [RCX];                        mov RCX, 8[RCX];                    L1: lock; cmpxchg16b [R8];                        jne L1;                        pop RBX;                        ret;                    }                }                else                {                    asm pure nothrow @nogc @trusted                    {                        naked;                        push RBX;                        mov RBX, RDI;                        mov RCX, RSI;                        mov RDI, RDX;                        mov RAX, [RDX];                        mov RDX, 8[RDX];                    L1: lock; cmpxchg16b [RDI];                        jne L1;                        pop RBX;                        ret;                    }                }            }        }        else static if (needsStoreBarrier!order)            atomicExchange!(order, false)(dest, value);        else            *dest = value;    }     T atomicFetchAdd(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted        if (is(T : ulong))    {        version (D_InlineAsm_X86)        {            static assert(T.sizeof <= 4, "64bit atomicFetchAdd not supported on 32bit target." );             enum DestReg = SizedReg!DX;            enum ValReg = SizedReg!(AX, T);             mixin (simpleFormat(q{                asm pure nothrow @nogc @trusted                {                    mov %1, value;                    mov %0, dest;                    lock; xadd[%0], %1;                }            }, [DestReg, ValReg]));        }        else version (D_InlineAsm_X86_64)        {            version (Windows)            {                enum DestReg = SizedReg!DX;                enum ValReg = SizedReg!(CX, T);            }            else            {                enum DestReg = SizedReg!SI;                enum ValReg = SizedReg!(DI, T);            }            enum ResReg = result ? SizedReg!(AX, T) : null;             mixin (simpleFormat(q{                asm pure nothrow @nogc @trusted                {                    naked;                    lock; xadd[%0], %1;    ?2                mov %2, %1;                    ret;                }            }, [DestReg, ValReg, ResReg]));        }        else            static assert (false, "Unsupported architecture.");    }     T atomicFetchSub(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted        if (is(T : ulong))    {        return atomicFetchAdd(dest, cast(T)-cast(IntOrLong!T)value);    }     T atomicExchange(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted    if (CanCAS!T)    {        static assert(order != MemoryOrder.acq, "Invalid MemoryOrder for atomicExchange()");        // We place some storage on the stack,        //  this storage and cast it to appropriete type.        // This is calling convention agnostic.        size_t storage = void;         version (D_InlineAsm_X86)        {            static assert(T.sizeof <= 4, "64bit atomicExchange not supported on 32bit target.");        }        else version (D_InlineAsm_X86_64)        {        }        else            static assert(0, "Operation not supported");         enum DestReg = SizedReg!CX;        enum ValReg = SizedReg!(AX, T);         mixin (simpleFormat(q{            asm pure nothrow @nogc @trusted            {                mov %1, value;                mov %0, dest;                lock; xchg [%0], %1;                 lea %0, storage;                mov [%0], %1;            }        }, [DestReg, ValReg]));         return *cast(T*)&storage;    }     alias atomicCompareExchangeWeak = atomicCompareExchangeStrong;     bool atomicCompareExchangeStrong(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted        if (CanCAS!T)    {        static assert(fail != MemoryOrder.rel && fail != MemoryOrder.acq_rel,                      "Invalid fail MemoryOrder for atomicCompareExchangeStrong()");        static assert (succ >= fail, "The first MemoryOrder argument for atomicCompareExchangeStrong() cannot be weaker than the second argument");        bool success;         static if (T.sizeof == size_t.sizeof * 2)        {            // some values simply cannot be loa'd here, so we'll use an intermediary pointer that we can move instead            T* valuePointer = &value;             version (D_InlineAsm_X86)            {                asm pure nothrow @nogc @trusted                {                    push EBX; // call preserved                    push EDI;                     mov EDI, valuePointer; // value                    mov EBX, [EDI];                    mov ECX, [EDI + size_t.sizeof];                    mov EDI, compare; // [compare]                    mov EAX, [EDI];                    mov EDX, [EDI + size_t.sizeof];                     mov EDI, dest;                    lock; cmpxchg8b [EDI];                     setz success;                    mov EDI, compare;                    mov [EDI], EAX;                    mov [EDI + size_t.sizeof], EDX;                     pop EDI;                    pop EBX;                }            }            else version (D_InlineAsm_X86_64)            {                asm pure nothrow @nogc @trusted                {                    push RBX; // call preserved                     mov R8, valuePointer; // value                    mov RBX, [R8];                    mov RCX, [R8 + size_t.sizeof];                    mov R8, compare; // [compare]                    mov RAX, [R8];                    mov RDX, [R8 + size_t.sizeof];                     mov R8, dest;                    lock; cmpxchg16b [R8];                     setz success;                    mov R8, compare;                    mov [R8], RAX;                    mov [R8 + size_t.sizeof], RDX;                     pop RBX;                }            }            else                static assert(0, "Operation not supported");        }        else        {            version (D_InlineAsm_X86)            {            }            else version (D_InlineAsm_X86_64)            {            }            else                static assert(0, "Operation not supported");             enum SrcReg = SizedReg!CX;            enum ValueReg = SizedReg!(DX, T);            enum CompareReg = SizedReg!(AX, T);             mixin (simpleFormat(q{                asm pure nothrow @nogc @trusted                {                    mov %1, value;                    mov %0, compare;                    mov %2, [%0];                     mov %0, dest;                    lock; cmpxchg [%0], %1;                     setz success;                    mov %0, compare;                    mov [%0], %2;                }            }, [SrcReg, ValueReg, CompareReg]));        }         return success;    }

Lines 1–500 of 1161. Load more