/* * A64 code generator for TCC * * Copyright (c) 2014-2015 Edmund Grimley Evans * * Copying and distribution of this file, with or without modification, * are permitted in any medium without royalty provided the copyright * notice and this notice are preserved. This file is offered as-is, * without any warranty. */ #ifdef TARGET_DEFS_ONLY // Number of registers available to allocator:#define NB_REGS 28 // x0-x18, x30, v0-v7 #define TREG_R(x) (x) // x = 0..18#define TREG_R30 19#define TREG_F(x) (x + 20) // x = 0..7 // Register classes sorted from more general to more precise:#define RC_INT (1 << 0)#define RC_FLOAT (1 << 1)#define RC_R(x) (1 << (2 + (x))) // x = 0..18#define RC_R30 (1 << 21)#define RC_F(x) (1 << (22 + (x))) // x = 0..7 #define RC_IRET (RC_R(0)) // int return register class#define RC_FRET (RC_F(0)) // float return register class #define REG_IRET (TREG_R(0)) // int return register number#define REG_FRET (TREG_F(0)) // float return register number #define PTR_SIZE 8 #define LDOUBLE_SIZE 16#define LDOUBLE_ALIGN 16 #define MAX_ALIGN 16 #define CHAR_IS_UNSIGNED /* define if return values need to be extended explicitely at caller side (for interfacing with non-TCC compilers) */#define PROMOTE_RET/******************************************************/#else /* ! TARGET_DEFS_ONLY *//******************************************************/#define USING_GLOBALS#include "tcc.h"#include <assert.h> ST_DATA const int reg_classes[NB_REGS] = { RC_INT | RC_R(0), RC_INT | RC_R(1), RC_INT | RC_R(2), RC_INT | RC_R(3), RC_INT | RC_R(4), RC_INT | RC_R(5), RC_INT | RC_R(6), RC_INT | RC_R(7), RC_INT | RC_R(8), RC_INT | RC_R(9), RC_INT | RC_R(10), RC_INT | RC_R(11), RC_INT | RC_R(12), RC_INT | RC_R(13), RC_INT | RC_R(14), RC_INT | RC_R(15), RC_INT | RC_R(16), RC_INT | RC_R(17), RC_INT | RC_R(18), RC_R30, // not in RC_INT as we make special use of x30 RC_FLOAT | RC_F(0), RC_FLOAT | RC_F(1), RC_FLOAT | RC_F(2), RC_FLOAT | RC_F(3), RC_FLOAT | RC_F(4), RC_FLOAT | RC_F(5), RC_FLOAT | RC_F(6), RC_FLOAT | RC_F(7)}; #if defined(CONFIG_TCC_BCHECK)static addr_t func_bound_offset;static unsigned long func_bound_ind;ST_DATA int func_bound_add_epilog;#endif #define IS_FREG(x) ((x) >= TREG_F(0)) static uint32_t intr(int r){ assert(TREG_R(0) <= r && r <= TREG_R30); return r < TREG_R30 ? r : 30;} static uint32_t fltr(int r){ assert(TREG_F(0) <= r && r <= TREG_F(7)); return r - TREG_F(0);} // Add an instruction to text section:ST_FUNC void o(unsigned int c){ int ind1 = ind + 4; if (nocode_wanted) return; if (ind1 > cur_text_section->data_allocated) section_realloc(cur_text_section, ind1); write32le(cur_text_section->data + ind, c); ind = ind1;} static int arm64_encode_bimm64(uint64_t x){ int neg = x & 1; int rep, pos, len; if (neg) x = ~x; if (!x) return -1; if (x >> 2 == (x & (((uint64_t)1 << (64 - 2)) - 1))) rep = 2, x &= ((uint64_t)1 << 2) - 1; else if (x >> 4 == (x & (((uint64_t)1 << (64 - 4)) - 1))) rep = 4, x &= ((uint64_t)1 << 4) - 1; else if (x >> 8 == (x & (((uint64_t)1 << (64 - 8)) - 1))) rep = 8, x &= ((uint64_t)1 << 8) - 1; else if (x >> 16 == (x & (((uint64_t)1 << (64 - 16)) - 1))) rep = 16, x &= ((uint64_t)1 << 16) - 1; else if (x >> 32 == (x & (((uint64_t)1 << (64 - 32)) - 1))) rep = 32, x &= ((uint64_t)1 << 32) - 1; else rep = 64; pos = 0; if (!(x & (((uint64_t)1 << 32) - 1))) x >>= 32, pos += 32; if (!(x & (((uint64_t)1 << 16) - 1))) x >>= 16, pos += 16; if (!(x & (((uint64_t)1 << 8) - 1))) x >>= 8, pos += 8; if (!(x & (((uint64_t)1 << 4) - 1))) x >>= 4, pos += 4; if (!(x & (((uint64_t)1 << 2) - 1))) x >>= 2, pos += 2; if (!(x & (((uint64_t)1 << 1) - 1))) x >>= 1, pos += 1; len = 0; if (!(~x & (((uint64_t)1 << 32) - 1))) x >>= 32, len += 32; if (!(~x & (((uint64_t)1 << 16) - 1))) x >>= 16, len += 16; if (!(~x & (((uint64_t)1 << 8) - 1))) x >>= 8, len += 8; if (!(~x & (((uint64_t)1 << 4) - 1))) x >>= 4, len += 4; if (!(~x & (((uint64_t)1 << 2) - 1))) x >>= 2, len += 2; if (!(~x & (((uint64_t)1 << 1) - 1))) x >>= 1, len += 1; if (x) return -1; if (neg) { pos = (pos + len) & (rep - 1); len = rep - len; } return ((0x1000 & rep << 6) | (((rep - 1) ^ 31) << 1 & 63) | ((rep - pos) & (rep - 1)) << 6 | (len - 1));} static uint32_t arm64_movi(int r, uint64_t x){ uint64_t m = 0xffff; int e; if (!(x & ~m)) return 0x52800000 | r | x << 5; // movz w(r),#(x) if (!(x & ~(m << 16))) return 0x52a00000 | r | x >> 11; // movz w(r),#(x >> 16),lsl #16 if (!(x & ~(m << 32))) return 0xd2c00000 | r | x >> 27; // movz x(r),#(x >> 32),lsl #32 if (!(x & ~(m << 48))) return 0xd2e00000 | r | x >> 43; // movz x(r),#(x >> 48),lsl #48 if ((x & ~m) == m << 16) return (0x12800000 | r | (~x << 5 & 0x1fffe0)); // movn w(r),#(~x) if ((x & ~(m << 16)) == m) return (0x12a00000 | r | (~x >> 11 & 0x1fffe0)); // movn w(r),#(~x >> 16),lsl #16 if (!~(x | m)) return (0x92800000 | r | (~x << 5 & 0x1fffe0)); // movn x(r),#(~x) if (!~(x | m << 16)) return (0x92a00000 | r | (~x >> 11 & 0x1fffe0)); // movn x(r),#(~x >> 16),lsl #16 if (!~(x | m << 32)) return (0x92c00000 | r | (~x >> 27 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32 if (!~(x | m << 48)) return (0x92e00000 | r | (~x >> 43 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32 if (!(x >> 32) && (e = arm64_encode_bimm64(x | x << 32)) >= 0) return 0x320003e0 | r | (uint32_t)e << 10; // movi w(r),#(x) if ((e = arm64_encode_bimm64(x)) >= 0) return 0xb20003e0 | r | (uint32_t)e << 10; // movi x(r),#(x) return 0;} static void arm64_movimm(int r, uint64_t x){ uint32_t i; if ((i = arm64_movi(r, x))) o(i); // a single MOV else { // MOVZ/MOVN and 1-3 MOVKs int z = 0, m = 0; uint32_t mov1 = 0xd2800000; // movz uint64_t x1 = x; for (i = 0; i < 64; i += 16) { z += !(x >> i & 0xffff); m += !(~x >> i & 0xffff); } if (m > z) { x1 = ~x; mov1 = 0x92800000; // movn } for (i = 0; i < 64; i += 16) if (x1 >> i & 0xffff) { o(mov1 | r | (x1 >> i & 0xffff) << 5 | i << 17); // movz/movn x(r),#(*),lsl #(i) break; } for (i += 16; i < 64; i += 16) if (x1 >> i & 0xffff) o(0xf2800000 | r | (x >> i & 0xffff) << 5 | i << 17); // movk x(r),#(*),lsl #(i) }} // Patch all branches in list pointed to by t to branch to a:ST_FUNC void gsym_addr(int t_, int a_){ uint32_t t = t_; uint32_t a = a_; while (t) { unsigned char *ptr = cur_text_section->data + t; uint32_t next = read32le(ptr); if (a - t + 0x8000000 >= 0x10000000) tcc_error("branch out of range"); write32le(ptr, (a - t == 4 ? 0xd503201f : // nop 0x14000000 | ((a - t) >> 2 & 0x3ffffff))); // b t = next; }} static int arm64_type_size(int t){ /* * case values are in increasing order (from 1 to 11). * which 'may' help compiler optimizers. See tcc.h */ switch (t & VT_BTYPE) { case VT_BYTE: return 0; case VT_SHORT: return 1; case VT_INT: return 2; case VT_LLONG: return 3; case VT_PTR: return 3; case VT_FUNC: return 3; case VT_STRUCT: return 3; case VT_FLOAT: return 2; case VT_DOUBLE: return 3; case VT_LDOUBLE: return 4; case VT_BOOL: return 0; } assert(0); return 0;} static void arm64_spoff(int reg, uint64_t off){ uint32_t sub = off >> 63; if (sub) off = -off; if (off < 4096) o(0x910003e0 | sub << 30 | reg | off << 10); // (add|sub) x(reg),sp,#(off) else { arm64_movimm(30, off); // use x30 for offset o(0x8b3e63e0 | sub << 30 | reg); // (add|sub) x(reg),sp,x30 }} /* invert 0: return value to use for store/load *//* invert 1: return value to use for arm64_sym */static uint64_t arm64_check_offset(int invert, int sz_, uint64_t off){ uint32_t sz = sz_; if (!(off & ~((uint32_t)0xfff << sz)) || (off < 256 || -off <= 256)) return invert ? off : 0ul; else if ((off & ((uint32_t)0xfff << sz))) return invert ? off & ((uint32_t)0xfff << sz) : off & ~((uint32_t)0xfff << sz); else if (off & 0x1ff) return invert ? off & 0x1ff : off & ~0x1ff; else return invert ? 0ul : off;} static void arm64_ldrx(int sg, int sz_, int dst, int bas, uint64_t off){ uint32_t sz = sz_; if (sz >= 2) sg = 0; if (!(off & ~((uint32_t)0xfff << sz))) o(0x39400000 | dst | bas << 5 | off << (10 - sz) | (uint32_t)!!sg << 23 | sz << 30); // ldr(*) x(dst),[x(bas),#(off)] else if (off < 256 || -off <= 256) o(0x38400000 | dst | bas << 5 | (off & 511) << 12 | (uint32_t)!!sg << 23 | sz << 30); // ldur(*) x(dst),[x(bas),#(off)] else { arm64_movimm(30, off); // use x30 for offset o(0x38206800 | dst | bas << 5 | (uint32_t)30 << 16 | (uint32_t)(!!sg + 1) << 22 | sz << 30); // ldr(*) x(dst),[x(bas),x30] }} static void arm64_ldrv(int sz_, int dst, int bas, uint64_t off){ uint32_t sz = sz_; if (!(off & ~((uint32_t)0xfff << sz))) o(0x3d400000 | dst | bas << 5 | off << (10 - sz) | (sz & 4) << 21 | (sz & 3) << 30); // ldr (s|d|q)(dst),[x(bas),#(off)] else if (off < 256 || -off <= 256) o(0x3c400000 | dst | bas << 5 | (off & 511) << 12 | (sz & 4) << 21 | (sz & 3) << 30); // ldur (s|d|q)(dst),[x(bas),#(off)] else { arm64_movimm(30, off); // use x30 for offset o(0x3c606800 | dst | bas << 5 | (uint32_t)30 << 16 | sz << 30 | (sz & 4) << 21); // ldr (s|d|q)(dst),[x(bas),x30] }} static void arm64_ldrs(int reg_, int size){ uint32_t reg = reg_; // Use x30 for intermediate value in some cases. switch (size) { default: assert(0); break; case 0: /* Can happen with zero size structs */ break; case 1: arm64_ldrx(0, 0, reg, reg, 0); break; case 2: arm64_ldrx(0, 1, reg, reg, 0); break; case 3: arm64_ldrx(0, 1, 30, reg, 0); arm64_ldrx(0, 0, reg, reg, 2); o(0x2a0043c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #16 break; case 4: arm64_ldrx(0, 2, reg, reg, 0); break; case 5: arm64_ldrx(0, 2, 30, reg, 0); arm64_ldrx(0, 0, reg, reg, 4); o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32 break; case 6: arm64_ldrx(0, 2, 30, reg, 0); arm64_ldrx(0, 1, reg, reg, 4); o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32 break; case 7: arm64_ldrx(0, 2, 30, reg, 0); arm64_ldrx(0, 2, reg, reg, 3); o(0x53087c00 | reg | reg << 5); // lsr w(reg), w(reg), #8 o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32 break; case 8: arm64_ldrx(0, 3, reg, reg, 0); break; case 9: arm64_ldrx(0, 0, reg + 1, reg, 8); arm64_ldrx(0, 3, reg, reg, 0); break; case 10: arm64_ldrx(0, 1, reg + 1, reg, 8); arm64_ldrx(0, 3, reg, reg, 0); break; case 11: arm64_ldrx(0, 2, reg + 1, reg, 7); o(0x53087c00 | (reg+1) | (reg+1) << 5); // lsr w(reg+1), w(reg+1), #8 arm64_ldrx(0, 3, reg, reg, 0); break; case 12: arm64_ldrx(0, 2, reg + 1, reg, 8); arm64_ldrx(0, 3, reg, reg, 0); break; case 13: arm64_ldrx(0, 3, reg + 1, reg, 5); o(0xd358fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #24 arm64_ldrx(0, 3, reg, reg, 0); break; case 14: arm64_ldrx(0, 3, reg + 1, reg, 6); o(0xd350fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #16 arm64_ldrx(0, 3, reg, reg, 0); break; case 15: arm64_ldrx(0, 3, reg + 1, reg, 7); o(0xd348fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #8 arm64_ldrx(0, 3, reg, reg, 0); break; case 16: o(0xa9400000 | reg | (reg+1) << 10 | reg << 5); // ldp x(reg),x(reg+1),[x(reg)] break; }} static void arm64_strx(int sz_, int dst, int bas, uint64_t off){ uint32_t sz = sz_; if (!(off & ~((uint32_t)0xfff << sz))) o(0x39000000 | dst | bas << 5 | off << (10 - sz) | sz << 30); // str(*) x(dst),[x(bas],#(off)] else if (off < 256 || -off <= 256) o(0x38000000 | dst | bas << 5 | (off & 511) << 12 | sz << 30); // stur(*) x(dst),[x(bas],#(off)] else { arm64_movimm(30, off); // use x30 for offset o(0x38206800 | dst | bas << 5 | (uint32_t)30 << 16 | sz << 30); // str(*) x(dst),[x(bas),x30] }} static void arm64_strv(int sz_, int dst, int bas, uint64_t off){ uint32_t sz = sz_; if (!(off & ~((uint32_t)0xfff << sz))) o(0x3d000000 | dst | bas << 5 | off << (10 - sz) | (sz & 4) << 21 | (sz & 3) << 30); // str (s|d|q)(dst),[x(bas),#(off)] else if (off < 256 || -off <= 256) o(0x3c000000 | dst | bas << 5 | (off & 511) << 12 | (sz & 4) << 21 | (sz & 3) << 30); // stur (s|d|q)(dst),[x(bas),#(off)] else { arm64_movimm(30, off); // use x30 for offset o(0x3c206800 | dst | bas << 5 | (uint32_t)30 << 16 | sz << 30 | (sz & 4) << 21); // str (s|d|q)(dst),[x(bas),x30] }} static void arm64_sym(int r, Sym *sym, unsigned long addend){ greloca(cur_text_section, sym, ind, R_AARCH64_ADR_GOT_PAGE, 0); o(0x90000000 | r); // adrp xr, #sym greloca(cur_text_section, sym, ind, R_AARCH64_LD64_GOT_LO12_NC, 0); o(0xf9400000 | r | (r << 5)); // ld xr,[xr, #sym] if (addend) { // add xr, xr, #addend if (addend & 0xffful) o(0x91000000 | r | r << 5 | (addend & 0xfff) << 10); if (addend > 0xffful) { // add xr, xr, #addend, lsl #12 if (addend & 0xfff000ul) o(0x91400000 | r | r << 5 | ((addend >> 12) & 0xfff) << 10); if (addend > 0xfffffful) { /* very unlikely */ int t = r ? 0 : 1; o(0xf81f0fe0 | t); /* str xt, [sp, #-16]! */ arm64_movimm(t, addend & ~0xfffffful); // use xt for addent o(0x91000000 | r | (t << 5)); /* add xr, xt, #0 */ o(0xf84107e0 | t); /* ldr xt, [sp], #16 */ } } }} static void arm64_load_cmp(int r, SValue *sv); ST_FUNC void load(int r, SValue *sv){ int svtt = sv->type.t; int svr = sv->r & ~VT_BOUNDED; int svrv = svr & VT_VALMASK; uint64_t svcul = (uint32_t)sv->c.i; svcul = svcul >> 31 & 1 ? svcul - ((uint64_t)1 << 32) : svcul; if (svr == (VT_LOCAL | VT_LVAL)) { if (IS_FREG(r)) arm64_ldrv(arm64_type_size(svtt), fltr(r), 29, svcul); else arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt), intr(r), 29, svcul); return; } if (svr == (VT_CONST | VT_LVAL)) { arm64_sym(30, sv->sym, // use x30 for address arm64_check_offset(0, arm64_type_size(svtt), sv->c.i)); if (IS_FREG(r)) arm64_ldrv(arm64_type_size(svtt), fltr(r), 30, arm64_check_offset(1, arm64_type_size(svtt), sv->c.i));Lines 1–500 of 2075. Load more
/* * A64 code generator for TCC * * Copyright (c) 2014-2015 Edmund Grimley Evans * * Copying and distribution of this file, with or without modification, * are permitted in any medium without royalty provided the copyright * notice and this notice are preserved. This file is offered as-is, * without any warranty. */ #ifdef TARGET_DEFS_ONLY // Number of registers available to allocator: #define NB_REGS 28 // x0-x18, x30, v0-v7 #define TREG_R(x) (x) // x = 0..18 #define TREG_R30 19 #define TREG_F(x) (x + 20) // x = 0..7 // Register classes sorted from more general to more precise: #define RC_INT (1 << 0) #define RC_FLOAT (1 << 1) #define RC_R(x) (1 << (2 + (x))) // x = 0..18 #define RC_R30 (1 << 21) #define RC_F(x) (1 << (22 + (x))) // x = 0..7 #define RC_IRET (RC_R(0)) // int return register class #define RC_FRET (RC_F(0)) // float return register class #define REG_IRET (TREG_R(0)) // int return register number #define REG_FRET (TREG_F(0)) // float return register number #define PTR_SIZE 8 #define LDOUBLE_SIZE 16 #define LDOUBLE_ALIGN 16 #define MAX_ALIGN 16 #define CHAR_IS_UNSIGNED /* define if return values need to be extended explicitely at caller side (for interfacing with non-TCC compilers) */ #define PROMOTE_RET /******************************************************/ #else /* ! TARGET_DEFS_ONLY */ /******************************************************/ #define USING_GLOBALS #include "tcc.h" #include <assert.h> ST_DATA const int reg_classes[NB_REGS] = { RC_INT | RC_R(0), RC_INT | RC_R(1), RC_INT | RC_R(2), RC_INT | RC_R(3), RC_INT | RC_R(4), RC_INT | RC_R(5), RC_INT | RC_R(6), RC_INT | RC_R(7), RC_INT | RC_R(8), RC_INT | RC_R(9), RC_INT | RC_R(10), RC_INT | RC_R(11), RC_INT | RC_R(12), RC_INT | RC_R(13), RC_INT | RC_R(14), RC_INT | RC_R(15), RC_INT | RC_R(16), RC_INT | RC_R(17), RC_INT | RC_R(18), RC_R30, // not in RC_INT as we make special use of x30 RC_FLOAT | RC_F(0), RC_FLOAT | RC_F(1), RC_FLOAT | RC_F(2), RC_FLOAT | RC_F(3), RC_FLOAT | RC_F(4), RC_FLOAT | RC_F(5), RC_FLOAT | RC_F(6), RC_FLOAT | RC_F(7) }; #if defined(CONFIG_TCC_BCHECK) static addr_t func_bound_offset; static unsigned long func_bound_ind; ST_DATA int func_bound_add_epilog; #endif #define IS_FREG(x) ((x) >= TREG_F(0)) static uint32_t intr(int r) { assert(TREG_R(0) <= r && r <= TREG_R30); return r < TREG_R30 ? r : 30; } static uint32_t fltr(int r) { assert(TREG_F(0) <= r && r <= TREG_F(7)); return r - TREG_F(0); } // Add an instruction to text section: ST_FUNC void o(unsigned int c) { int ind1 = ind + 4; if (nocode_wanted) return; if (ind1 > cur_text_section->data_allocated) section_realloc(cur_text_section, ind1); write32le(cur_text_section->data + ind, c); ind = ind1; } static int arm64_encode_bimm64(uint64_t x) { int neg = x & 1; int rep, pos, len; if (neg) x = ~x; if (!x) return -1; if (x >> 2 == (x & (((uint64_t)1 << (64 - 2)) - 1))) rep = 2, x &= ((uint64_t)1 << 2) - 1; else if (x >> 4 == (x & (((uint64_t)1 << (64 - 4)) - 1))) rep = 4, x &= ((uint64_t)1 << 4) - 1; else if (x >> 8 == (x & (((uint64_t)1 << (64 - 8)) - 1))) rep = 8, x &= ((uint64_t)1 << 8) - 1; else if (x >> 16 == (x & (((uint64_t)1 << (64 - 16)) - 1))) rep = 16, x &= ((uint64_t)1 << 16) - 1; else if (x >> 32 == (x & (((uint64_t)1 << (64 - 32)) - 1))) rep = 32, x &= ((uint64_t)1 << 32) - 1; else rep = 64; pos = 0; if (!(x & (((uint64_t)1 << 32) - 1))) x >>= 32, pos += 32; if (!(x & (((uint64_t)1 << 16) - 1))) x >>= 16, pos += 16; if (!(x & (((uint64_t)1 << 8) - 1))) x >>= 8, pos += 8; if (!(x & (((uint64_t)1 << 4) - 1))) x >>= 4, pos += 4; if (!(x & (((uint64_t)1 << 2) - 1))) x >>= 2, pos += 2; if (!(x & (((uint64_t)1 << 1) - 1))) x >>= 1, pos += 1; len = 0; if (!(~x & (((uint64_t)1 << 32) - 1))) x >>= 32, len += 32; if (!(~x & (((uint64_t)1 << 16) - 1))) x >>= 16, len += 16; if (!(~x & (((uint64_t)1 << 8) - 1))) x >>= 8, len += 8; if (!(~x & (((uint64_t)1 << 4) - 1))) x >>= 4, len += 4; if (!(~x & (((uint64_t)1 << 2) - 1))) x >>= 2, len += 2; if (!(~x & (((uint64_t)1 << 1) - 1))) x >>= 1, len += 1; if (x) return -1; if (neg) { pos = (pos + len) & (rep - 1); len = rep - len; } return ((0x1000 & rep << 6) | (((rep - 1) ^ 31) << 1 & 63) | ((rep - pos) & (rep - 1)) << 6 | (len - 1)); } static uint32_t arm64_movi(int r, uint64_t x) { uint64_t m = 0xffff; int e; if (!(x & ~m)) return 0x52800000 | r | x << 5; // movz w(r),#(x) if (!(x & ~(m << 16))) return 0x52a00000 | r | x >> 11; // movz w(r),#(x >> 16),lsl #16 if (!(x & ~(m << 32))) return 0xd2c00000 | r | x >> 27; // movz x(r),#(x >> 32),lsl #32 if (!(x & ~(m << 48))) return 0xd2e00000 | r | x >> 43; // movz x(r),#(x >> 48),lsl #48 if ((x & ~m) == m << 16) return (0x12800000 | r | (~x << 5 & 0x1fffe0)); // movn w(r),#(~x) if ((x & ~(m << 16)) == m) return (0x12a00000 | r | (~x >> 11 & 0x1fffe0)); // movn w(r),#(~x >> 16),lsl #16 if (!~(x | m)) return (0x92800000 | r | (~x << 5 & 0x1fffe0)); // movn x(r),#(~x) if (!~(x | m << 16)) return (0x92a00000 | r | (~x >> 11 & 0x1fffe0)); // movn x(r),#(~x >> 16),lsl #16 if (!~(x | m << 32)) return (0x92c00000 | r | (~x >> 27 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32 if (!~(x | m << 48)) return (0x92e00000 | r | (~x >> 43 & 0x1fffe0)); // movn x(r),#(~x >> 32),lsl #32 if (!(x >> 32) && (e = arm64_encode_bimm64(x | x << 32)) >= 0) return 0x320003e0 | r | (uint32_t)e << 10; // movi w(r),#(x) if ((e = arm64_encode_bimm64(x)) >= 0) return 0xb20003e0 | r | (uint32_t)e << 10; // movi x(r),#(x) return 0; } static void arm64_movimm(int r, uint64_t x) { uint32_t i; if ((i = arm64_movi(r, x))) o(i); // a single MOV else { // MOVZ/MOVN and 1-3 MOVKs int z = 0, m = 0; uint32_t mov1 = 0xd2800000; // movz uint64_t x1 = x; for (i = 0; i < 64; i += 16) { z += !(x >> i & 0xffff); m += !(~x >> i & 0xffff); } if (m > z) { x1 = ~x; mov1 = 0x92800000; // movn } for (i = 0; i < 64; i += 16) if (x1 >> i & 0xffff) { o(mov1 | r | (x1 >> i & 0xffff) << 5 | i << 17); // movz/movn x(r),#(*),lsl #(i) break; } for (i += 16; i < 64; i += 16) if (x1 >> i & 0xffff) o(0xf2800000 | r | (x >> i & 0xffff) << 5 | i << 17); // movk x(r),#(*),lsl #(i) } } // Patch all branches in list pointed to by t to branch to a: ST_FUNC void gsym_addr(int t_, int a_) { uint32_t t = t_; uint32_t a = a_; while (t) { unsigned char *ptr = cur_text_section->data + t; uint32_t next = read32le(ptr); if (a - t + 0x8000000 >= 0x10000000) tcc_error("branch out of range"); write32le(ptr, (a - t == 4 ? 0xd503201f : // nop 0x14000000 | ((a - t) >> 2 & 0x3ffffff))); // b t = next; } } static int arm64_type_size(int t) { /* * case values are in increasing order (from 1 to 11). * which 'may' help compiler optimizers. See tcc.h */ switch (t & VT_BTYPE) { case VT_BYTE: return 0; case VT_SHORT: return 1; case VT_INT: return 2; case VT_LLONG: return 3; case VT_PTR: return 3; case VT_FUNC: return 3; case VT_STRUCT: return 3; case VT_FLOAT: return 2; case VT_DOUBLE: return 3; case VT_LDOUBLE: return 4; case VT_BOOL: return 0; } assert(0); return 0; } static void arm64_spoff(int reg, uint64_t off) { uint32_t sub = off >> 63; if (sub) off = -off; if (off < 4096) o(0x910003e0 | sub << 30 | reg | off << 10); // (add|sub) x(reg),sp,#(off) else { arm64_movimm(30, off); // use x30 for offset o(0x8b3e63e0 | sub << 30 | reg); // (add|sub) x(reg),sp,x30 } } /* invert 0: return value to use for store/load */ /* invert 1: return value to use for arm64_sym */ static uint64_t arm64_check_offset(int invert, int sz_, uint64_t off) { uint32_t sz = sz_; if (!(off & ~((uint32_t)0xfff << sz)) || (off < 256 || -off <= 256)) return invert ? off : 0ul; else if ((off & ((uint32_t)0xfff << sz))) return invert ? off & ((uint32_t)0xfff << sz) : off & ~((uint32_t)0xfff << sz); else if (off & 0x1ff) return invert ? off & 0x1ff : off & ~0x1ff; else return invert ? 0ul : off; } static void arm64_ldrx(int sg, int sz_, int dst, int bas, uint64_t off) { uint32_t sz = sz_; if (sz >= 2) sg = 0; if (!(off & ~((uint32_t)0xfff << sz))) o(0x39400000 | dst | bas << 5 | off << (10 - sz) | (uint32_t)!!sg << 23 | sz << 30); // ldr(*) x(dst),[x(bas),#(off)] else if (off < 256 || -off <= 256) o(0x38400000 | dst | bas << 5 | (off & 511) << 12 | (uint32_t)!!sg << 23 | sz << 30); // ldur(*) x(dst),[x(bas),#(off)] else { arm64_movimm(30, off); // use x30 for offset o(0x38206800 | dst | bas << 5 | (uint32_t)30 << 16 | (uint32_t)(!!sg + 1) << 22 | sz << 30); // ldr(*) x(dst),[x(bas),x30] } } static void arm64_ldrv(int sz_, int dst, int bas, uint64_t off) { uint32_t sz = sz_; if (!(off & ~((uint32_t)0xfff << sz))) o(0x3d400000 | dst | bas << 5 | off << (10 - sz) | (sz & 4) << 21 | (sz & 3) << 30); // ldr (s|d|q)(dst),[x(bas),#(off)] else if (off < 256 || -off <= 256) o(0x3c400000 | dst | bas << 5 | (off & 511) << 12 | (sz & 4) << 21 | (sz & 3) << 30); // ldur (s|d|q)(dst),[x(bas),#(off)] else { arm64_movimm(30, off); // use x30 for offset o(0x3c606800 | dst | bas << 5 | (uint32_t)30 << 16 | sz << 30 | (sz & 4) << 21); // ldr (s|d|q)(dst),[x(bas),x30] } } static void arm64_ldrs(int reg_, int size) { uint32_t reg = reg_; // Use x30 for intermediate value in some cases. switch (size) { default: assert(0); break; case 0: /* Can happen with zero size structs */ break; case 1: arm64_ldrx(0, 0, reg, reg, 0); break; case 2: arm64_ldrx(0, 1, reg, reg, 0); break; case 3: arm64_ldrx(0, 1, 30, reg, 0); arm64_ldrx(0, 0, reg, reg, 2); o(0x2a0043c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #16 break; case 4: arm64_ldrx(0, 2, reg, reg, 0); break; case 5: arm64_ldrx(0, 2, 30, reg, 0); arm64_ldrx(0, 0, reg, reg, 4); o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32 break; case 6: arm64_ldrx(0, 2, 30, reg, 0); arm64_ldrx(0, 1, reg, reg, 4); o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32 break; case 7: arm64_ldrx(0, 2, 30, reg, 0); arm64_ldrx(0, 2, reg, reg, 3); o(0x53087c00 | reg | reg << 5); // lsr w(reg), w(reg), #8 o(0xaa0083c0 | reg | reg << 16); // orr x(reg),x30,x(reg),lsl #32 break; case 8: arm64_ldrx(0, 3, reg, reg, 0); break; case 9: arm64_ldrx(0, 0, reg + 1, reg, 8); arm64_ldrx(0, 3, reg, reg, 0); break; case 10: arm64_ldrx(0, 1, reg + 1, reg, 8); arm64_ldrx(0, 3, reg, reg, 0); break; case 11: arm64_ldrx(0, 2, reg + 1, reg, 7); o(0x53087c00 | (reg+1) | (reg+1) << 5); // lsr w(reg+1), w(reg+1), #8 arm64_ldrx(0, 3, reg, reg, 0); break; case 12: arm64_ldrx(0, 2, reg + 1, reg, 8); arm64_ldrx(0, 3, reg, reg, 0); break; case 13: arm64_ldrx(0, 3, reg + 1, reg, 5); o(0xd358fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #24 arm64_ldrx(0, 3, reg, reg, 0); break; case 14: arm64_ldrx(0, 3, reg + 1, reg, 6); o(0xd350fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #16 arm64_ldrx(0, 3, reg, reg, 0); break; case 15: arm64_ldrx(0, 3, reg + 1, reg, 7); o(0xd348fc00 | (reg+1) | (reg+1) << 5); // lsr x(reg+1), x(reg+1), #8 arm64_ldrx(0, 3, reg, reg, 0); break; case 16: o(0xa9400000 | reg | (reg+1) << 10 | reg << 5); // ldp x(reg),x(reg+1),[x(reg)] break; } } static void arm64_strx(int sz_, int dst, int bas, uint64_t off) { uint32_t sz = sz_; if (!(off & ~((uint32_t)0xfff << sz))) o(0x39000000 | dst | bas << 5 | off << (10 - sz) | sz << 30); // str(*) x(dst),[x(bas],#(off)] else if (off < 256 || -off <= 256) o(0x38000000 | dst | bas << 5 | (off & 511) << 12 | sz << 30); // stur(*) x(dst),[x(bas],#(off)] else { arm64_movimm(30, off); // use x30 for offset o(0x38206800 | dst | bas << 5 | (uint32_t)30 << 16 | sz << 30); // str(*) x(dst),[x(bas),x30] } } static void arm64_strv(int sz_, int dst, int bas, uint64_t off) { uint32_t sz = sz_; if (!(off & ~((uint32_t)0xfff << sz))) o(0x3d000000 | dst | bas << 5 | off << (10 - sz) | (sz & 4) << 21 | (sz & 3) << 30); // str (s|d|q)(dst),[x(bas),#(off)] else if (off < 256 || -off <= 256) o(0x3c000000 | dst | bas << 5 | (off & 511) << 12 | (sz & 4) << 21 | (sz & 3) << 30); // stur (s|d|q)(dst),[x(bas),#(off)] else { arm64_movimm(30, off); // use x30 for offset o(0x3c206800 | dst | bas << 5 | (uint32_t)30 << 16 | sz << 30 | (sz & 4) << 21); // str (s|d|q)(dst),[x(bas),x30] } } static void arm64_sym(int r, Sym *sym, unsigned long addend) { greloca(cur_text_section, sym, ind, R_AARCH64_ADR_GOT_PAGE, 0); o(0x90000000 | r); // adrp xr, #sym greloca(cur_text_section, sym, ind, R_AARCH64_LD64_GOT_LO12_NC, 0); o(0xf9400000 | r | (r << 5)); // ld xr,[xr, #sym] if (addend) { // add xr, xr, #addend if (addend & 0xffful) o(0x91000000 | r | r << 5 | (addend & 0xfff) << 10); if (addend > 0xffful) { // add xr, xr, #addend, lsl #12 if (addend & 0xfff000ul) o(0x91400000 | r | r << 5 | ((addend >> 12) & 0xfff) << 10); if (addend > 0xfffffful) { /* very unlikely */ int t = r ? 0 : 1; o(0xf81f0fe0 | t); /* str xt, [sp, #-16]! */ arm64_movimm(t, addend & ~0xfffffful); // use xt for addent o(0x91000000 | r | (t << 5)); /* add xr, xt, #0 */ o(0xf84107e0 | t); /* ldr xt, [sp], #16 */ } } } } static void arm64_load_cmp(int r, SValue *sv); ST_FUNC void load(int r, SValue *sv) { int svtt = sv->type.t; int svr = sv->r & ~VT_BOUNDED; int svrv = svr & VT_VALMASK; uint64_t svcul = (uint32_t)sv->c.i; svcul = svcul >> 31 & 1 ? svcul - ((uint64_t)1 << 32) : svcul; if (svr == (VT_LOCAL | VT_LVAL)) { if (IS_FREG(r)) arm64_ldrv(arm64_type_size(svtt), fltr(r), 29, svcul); else arm64_ldrx(!(svtt & VT_UNSIGNED), arm64_type_size(svtt), intr(r), 29, svcul); return; } if (svr == (VT_CONST | VT_LVAL)) { arm64_sym(30, sv->sym, // use x30 for address arm64_check_offset(0, arm64_type_size(svtt), sv->c.i)); if (IS_FREG(r)) arm64_ldrv(arm64_type_size(svtt), fltr(r), 30, arm64_check_offset(1, arm64_type_size(svtt), sv->c.i));