Also made XMM0-7 available for use as temporary registers, since they are not used by the ABI. I'd like to do the same with RSI and RDI but that's trickier since they can be used by gv() as temporary registers and there isn't a way to disable that.
Unified · Split
| @@ -764,7 +764,7 @@ ST_FUNC int gv(int rc) · 763 unmodified lines | |||
| 764 | #endif | 764 | #endif |
| 765 | 765 | ||
| 766 | r = vtop->r & VT_VALMASK; | 766 | r = vtop->r & VT_VALMASK; |
| 767 | rc2 = RC_INT; | 767 | rc2 = (rc & RC_FLOAT) ? RC_FLOAT : RC_INT; |
| 768 | if (rc == RC_IRET) | 768 | if (rc == RC_IRET) |
| 769 | rc2 = RC_LRET; | 769 | rc2 = RC_LRET; |
| 770 | #ifdef TCC_TARGET_X86_64 | 770 | #ifdef TCC_TARGET_X86_64 |
| @@ -23,7 +23,7 @@ · 22 unmodified lines | |||
| 23 | #ifdef TARGET_DEFS_ONLY | 23 | #ifdef TARGET_DEFS_ONLY |
| 24 | 24 | ||
| 25 | /* number of available registers */ | 25 | /* number of available registers */ |
| 26 | #define NB_REGS 18 | 26 | #define NB_REGS 24 |
| 27 | #define NB_ASM_REGS 8 | 27 | #define NB_ASM_REGS 8 |
| 28 | 28 | ||
| 29 | /* a register can belong to several classes. The classes must be | 29 | /* a register can belong to several classes. The classes must be |
| @@ -34,13 +34,19 @@ · 4 unmodified lines | |||
| 34 | #define RC_RAX 0x0004 | 34 | #define RC_RAX 0x0004 |
| 35 | #define RC_RCX 0x0008 | 35 | #define RC_RCX 0x0008 |
| 36 | #define RC_RDX 0x0010 | 36 | #define RC_RDX 0x0010 |
| 37 | #define RC_ST0 0x0080 /* only for long double */ | ||
| 37 | #define RC_R8 0x0100 | 38 | #define RC_R8 0x0100 |
| 38 | #define RC_R9 0x0200 | 39 | #define RC_R9 0x0200 |
| 39 | #define RC_R10 0x0400 | 40 | #define RC_R10 0x0400 |
| 40 | #define RC_R11 0x0800 | 41 | #define RC_R11 0x0800 |
| 41 | #define RC_XMM0 0x0020 | 42 | #define RC_XMM0 0x1000 |
| 42 | #define RC_XMM1 0x0040 | 43 | #define RC_XMM1 0x2000 |
| 43 | #define RC_ST0 0x0080 /* only for long double */ | 44 | #define RC_XMM2 0x4000 |
| 45 | #define RC_XMM3 0x8000 | ||
| 46 | #define RC_XMM4 0x10000 | ||
| 47 | #define RC_XMM5 0x20000 | ||
| 48 | #define RC_XMM6 0x40000 | ||
| 49 | #define RC_XMM7 0x80000 | ||
| 44 | #define RC_IRET RC_RAX /* function return: integer register */ | 50 | #define RC_IRET RC_RAX /* function return: integer register */ |
| 45 | #define RC_LRET RC_RDX /* function return: second integer register */ | 51 | #define RC_LRET RC_RDX /* function return: second integer register */ |
| 46 | #define RC_FRET RC_XMM0 /* function return: float register */ | 52 | #define RC_FRET RC_XMM0 /* function return: float register */ |
| @@ -61,6 +67,12 @@ enum { · 14 unmodified lines | |||
| 61 | 67 | ||
| 62 | TREG_XMM0 = 16, | 68 | TREG_XMM0 = 16, |
| 63 | TREG_XMM1 = 17, | 69 | TREG_XMM1 = 17, |
| 70 | TREG_XMM2 = 18, | ||
| 71 | TREG_XMM3 = 19, | ||
| 72 | TREG_XMM4 = 20, | ||
| 73 | TREG_XMM5 = 21, | ||
| 74 | TREG_XMM6 = 22, | ||
| 75 | TREG_XMM7 = 23, | ||
| 64 | 76 | ||
| 65 | TREG_ST0 = 4, // SP slot won't be used | 77 | TREG_ST0 = 4, // SP slot won't be used |
| 66 | 78 | ||
| @@ -117,16 +129,22 @@ ST_DATA const int reg_classes[NB_REGS] = { · 50 unmodified lines | |||
| 117 | 0, | 129 | 0, |
| 118 | 0, | 130 | 0, |
| 119 | 0, | 131 | 0, |
| 120 | /*RC_INT |*/ RC_R8, | 132 | RC_R8, |
| 121 | /*RC_INT |*/ RC_R9, | 133 | RC_R9, |
| 122 | /*RC_INT |*/ RC_R10, | 134 | RC_R10, |
| 123 | /*RC_INT |*/ RC_R11, | 135 | RC_R11, |
| 124 | 0, | 136 | 0, |
| 125 | 0, | 137 | 0, |
| 126 | 0, | 138 | 0, |
| 127 | 0, | 139 | 0, |
| 128 | /* xmm0 */ RC_FLOAT | RC_XMM0, | 140 | /* xmm0 */ RC_FLOAT | RC_XMM0, |
| 129 | /* xmm1 */ RC_FLOAT | RC_XMM1, | 141 | /* xmm1 */ RC_FLOAT | RC_XMM1, |
| 142 | /* xmm2 */ RC_FLOAT | RC_XMM2, | ||
| 143 | /* xmm3 */ RC_FLOAT | RC_XMM3, | ||
| 144 | /* xmm4 */ RC_FLOAT | RC_XMM4, | ||
| 145 | /* xmm5 */ RC_FLOAT | RC_XMM5, | ||
| 146 | /* xmm6 */ RC_FLOAT | RC_XMM6, | ||
| 147 | /* xmm7 */ RC_FLOAT | RC_XMM7, | ||
| 130 | }; | 148 | }; |
| 131 | 149 | ||
| 132 | static unsigned long func_sub_sp_offset; | 150 | static unsigned long func_sub_sp_offset; |
| @@ -141,8 +159,6 @@ void g(int c) · 8 unmodified lines | |||
| 141 | section_realloc(cur_text_section, ind1); | 159 | section_realloc(cur_text_section, ind1); |
| 142 | cur_text_section->data[ind] = c; | 160 | cur_text_section->data[ind] = c; |
| 143 | ind = ind1; | 161 | ind = ind1; |
| 144 | assert((ind < 4) || (cur_text_section->data[ind-4] != ('\362'&0xFF)) || (cur_text_section->data[ind-3] != '\017') | ||
| 145 | || (cur_text_section->data[ind-2] != 'X') || (cur_text_section->data[ind-1] != '\001')); | ||
| 146 | } | 162 | } |
| 147 | 163 | ||
| 148 | void o(unsigned int c) | 164 | void o(unsigned int c) |
| @@ -1055,32 +1071,48 @@ void gfunc_call(int nb_args) · 906 unmodified lines | |||
| 1055 | oad(0xec81, args_size); /* sub $xxx, %rsp */ | 1071 | oad(0xec81, args_size); /* sub $xxx, %rsp */ |
| 1056 | } | 1072 | } |
| 1057 | 1073 | ||
| 1058 | for(i = 0; i < nb_args; i++) { | 1074 | for(i = 0; i < nb_args;) { |
| 1059 | /* Swap argument to top, it will possibly be changed here, | 1075 | /* Swap argument to top, it will possibly be changed here, |
| 1060 | and might use more temps. All arguments must remain on the | 1076 | and might use more temps. At the end of the loop we keep |
| 1061 | stack, so that get_reg can correctly evict some of them onto | 1077 | in on the stack and swap it back to its original position |
| 1062 | stack. We could use also use a vrott(nb_args) at the end | 1078 | if it is a register. */ |
| 1063 | of this loop, but this seems faster. */ | ||
| 1064 | SValue tmp = vtop[0]; | 1079 | SValue tmp = vtop[0]; |
| 1065 | vtop[0] = vtop[-i]; | 1080 | vtop[0] = vtop[-i]; |
| 1066 | vtop[-i] = tmp; | 1081 | vtop[-i] = tmp; |
| 1082 | |||
| 1067 | mode = classify_x86_64_arg(&vtop->type, NULL, &size, ®_count); | 1083 | mode = classify_x86_64_arg(&vtop->type, NULL, &size, ®_count); |
| 1068 | switch (mode) { | 1084 | |
| 1069 | case x86_64_mode_memory: | 1085 | int arg_stored = 1; |
| 1070 | /* allocate the necessary size on stack */ | 1086 | switch (vtop->type.t & VT_BTYPE) { |
| 1071 | o(0x48); | 1087 | case VT_STRUCT: |
| 1072 | oad(0xec81, size); /* sub $xxx, %rsp */ | 1088 | if (mode == x86_64_mode_sse) { |
| 1073 | /* generate structure store */ | 1089 | if (sse_reg > 8) |
| 1074 | r = get_reg(RC_INT); | 1090 | sse_reg -= reg_count; |
| 1075 | orex(1, r, 0, 0x89); /* mov %rsp, r */ | 1091 | else |
| 1076 | o(0xe0 + REG_VALUE(r)); | 1092 | arg_stored = 0; |
| 1077 | vset(&vtop->type, r | VT_LVAL, 0); | 1093 | } else if (mode == x86_64_mode_integer) { |
| 1078 | vswap(); | 1094 | if (gen_reg > REGN) |
| 1079 | vstore(); | 1095 | gen_reg -= reg_count; |
| 1080 | args_size += size; | 1096 | else |
| 1097 | arg_stored = 0; | ||
| 1098 | } | ||
| 1099 | |||
| 1100 | if (arg_stored) { | ||
| 1101 | /* allocate the necessary size on stack */ | ||
| 1102 | o(0x48); | ||
| 1103 | oad(0xec81, size); /* sub $xxx, %rsp */ | ||
| 1104 | /* generate structure store */ | ||
| 1105 | r = get_reg(RC_INT); | ||
| 1106 | orex(1, r, 0, 0x89); /* mov %rsp, r */ | ||
| 1107 | o(0xe0 + REG_VALUE(r)); | ||
| 1108 | vset(&vtop->type, r | VT_LVAL, 0); | ||
| 1109 | vswap(); | ||
| 1110 | vstore(); | ||
| 1111 | args_size += size; | ||
| 1112 | } | ||
| 1081 | break; | 1113 | break; |
| 1082 | 1114 | ||
| 1083 | case x86_64_mode_x87: | 1115 | case VT_LDOUBLE: |
| 1084 | gv(RC_ST0); | 1116 | gv(RC_ST0); |
| 1085 | size = LDOUBLE_SIZE; | 1117 | size = LDOUBLE_SIZE; |
| 1086 | oad(0xec8148, size); /* sub $xxx, %rsp */ | 1118 | oad(0xec8148, size); /* sub $xxx, %rsp */ |
| @@ -1090,8 +1122,11 @@ void gfunc_call(int nb_args) · 3 unmodified lines | |||
| 1090 | args_size += size; | 1122 | args_size += size; |
| 1091 | break; | 1123 | break; |
| 1092 | 1124 | ||
| 1093 | case x86_64_mode_sse: | 1125 | case VT_FLOAT: |
| 1126 | case VT_DOUBLE: | ||
| 1127 | assert(mode == x86_64_mode_sse); | ||
| 1094 | if (sse_reg > 8) { | 1128 | if (sse_reg > 8) { |
| 1129 | --sse_reg; | ||
| 1095 | r = gv(RC_FLOAT); | 1130 | r = gv(RC_FLOAT); |
| 1096 | o(0x50); /* push $rax */ | 1131 | o(0x50); /* push $rax */ |
| 1097 | /* movq %xmm0, (%rsp) */ | 1132 | /* movq %xmm0, (%rsp) */ |
| @@ -1099,26 +1134,39 @@ void gfunc_call(int nb_args) · 1 unmodified line | |||
| 1099 | o(0x04 + REG_VALUE(r)*8); | 1134 | o(0x04 + REG_VALUE(r)*8); |
| 1100 | o(0x24); | 1135 | o(0x24); |
| 1101 | args_size += size; | 1136 | args_size += size; |
| 1137 | } else { | ||
| 1138 | arg_stored = 0; | ||
| 1102 | } | 1139 | } |
| 1103 | sse_reg -= reg_count; | ||
| 1104 | break; | 1140 | break; |
| 1105 | 1141 | ||
| 1106 | case x86_64_mode_integer: | 1142 | default: |
| 1143 | assert(mode == x86_64_mode_integer); | ||
| 1107 | /* simple type */ | 1144 | /* simple type */ |
| 1108 | /* XXX: implicit cast ? */ | 1145 | /* XXX: implicit cast ? */ |
| 1109 | if (gen_reg > REGN) { | 1146 | if (gen_reg > REGN) { |
| 1147 | --gen_reg; | ||
| 1110 | r = gv(RC_INT); | 1148 | r = gv(RC_INT); |
| 1111 | orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */ | 1149 | orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */ |
| 1112 | args_size += size; | 1150 | args_size += size; |
| 1151 | } else { | ||
| 1152 | arg_stored = 0; | ||
| 1113 | } | 1153 | } |
| 1114 | gen_reg -= reg_count; | ||
| 1115 | break; | 1154 | break; |
| 1116 | } | 1155 | } |
| 1117 | 1156 | ||
| 1118 | /* And swap the argument back to it's original position. */ | 1157 | /* And swap the argument back to it's original position. */ |
| 1119 | tmp = vtop[0]; | 1158 | tmp = vtop[0]; |
| 1120 | vtop[0] = vtop[-i]; | 1159 | vtop[0] = vtop[-i]; |
| 1121 | vtop[-i] = tmp; | 1160 | vtop[-i] = tmp; |
| 1161 | |||
| 1162 | if (arg_stored) { | ||
| 1163 | vrotb(i+1); | ||
| 1164 | assert(vtop->type.t == tmp.type.t); | ||
| 1165 | vpop(); | ||
| 1166 | --nb_args; | ||
| 1167 | } else { | ||
| 1168 | ++i; | ||
| 1169 | } | ||
| 1122 | } | 1170 | } |
| 1123 | 1171 | ||
| 1124 | /* XXX This should be superfluous. */ | 1172 | /* XXX This should be superfluous. */ |
| @@ -1128,55 +1176,50 @@ void gfunc_call(int nb_args) · 3 unmodified lines | |||
| 1128 | Note that we cannot set RDX and RCX in this loop because gv() | 1176 | Note that we cannot set RDX and RCX in this loop because gv() |
| 1129 | may break these temporary registers. Let's use R10 and R11 | 1177 | may break these temporary registers. Let's use R10 and R11 |
| 1130 | instead of them */ | 1178 | instead of them */ |
| 1131 | gen_reg = nb_reg_args; | 1179 | assert(gen_reg <= REGN); |
| 1132 | sse_reg = nb_sse_args; | 1180 | assert(sse_reg <= 8); |
| 1133 | for(i = 0; i < nb_args; i++) { | 1181 | for(i = 0; i < nb_args; i++) { |
| 1134 | mode = classify_x86_64_arg(&vtop->type, &type, &size, ®_count); | 1182 | mode = classify_x86_64_arg(&vtop->type, &type, &size, ®_count); |
| 1135 | /* Alter stack entry type so that gv() knows how to treat it */ | 1183 | /* Alter stack entry type so that gv() knows how to treat it */ |
| 1136 | vtop->type = type; | 1184 | vtop->type = type; |
| 1137 | switch (mode) { | 1185 | if (mode == x86_64_mode_sse) { |
| 1138 | default: | 1186 | if (reg_count == 2) { |
| 1139 | break; | 1187 | sse_reg -= 2; |
| 1140 | 1188 | gv(RC_FRET); /* Use pair load into xmm0 & xmm1 */ | |
| 1141 | case x86_64_mode_sse: | 1189 | if (sse_reg) { /* avoid redundant movaps %xmm0, %xmm0 */ |
| 1142 | sse_reg -= reg_count; | 1190 | /* movaps %xmm0, %xmmN */ |
| 1143 | if (sse_reg + reg_count <= 8) { | 1191 | o(0x280f); |
| 1144 | gv(RC_FRET); /* only one float register */ | 1192 | o(0xc0 + (sse_reg << 3)); |
| 1145 | if (sse_reg) { /* avoid redundant movaps %xmm0, %xmm0 */ | 1193 | /* movaps %xmm1, %xmmN */ |
| 1146 | /* movaps %xmm0, %xmmN */ | 1194 | o(0x280f); |
| 1147 | o(0x280f); | 1195 | o(0xc1 + ((sse_reg+1) << 3)); |
| 1148 | o(0xc0 + (sse_reg << 3)); | 1196 | } |
| 1149 | if (reg_count == 2) { | 1197 | } else { |
| 1150 | /* movaps %xmm1, %xmmN */ | 1198 | assert(reg_count == 1); |
| 1151 | o(0x280f); | 1199 | --sse_reg; |
| 1152 | o(0xc1 + ((sse_reg+1) << 3)); | 1200 | /* Load directly to register */ |
| 1153 | } | 1201 | gv(RC_XMM0 << sse_reg); |
| 1154 | } | 1202 | } |
| 1155 | } | 1203 | } else if (mode == x86_64_mode_integer) { |
| 1156 | break; | ||
| 1157 | |||
| 1158 | case x86_64_mode_integer: | ||
| 1159 | /* simple type */ | 1204 | /* simple type */ |
| 1160 | /* XXX: implicit cast ? */ | 1205 | /* XXX: implicit cast ? */ |
| 1161 | gen_reg -= reg_count; | 1206 | gen_reg -= reg_count; |
| 1162 | if (gen_reg + reg_count <= REGN) { | 1207 | r = gv(RC_INT); |
| 1163 | r = gv((reg_count == 1) ? RC_INT : RC_IRET); | 1208 | int d = arg_prepare_reg(gen_reg); |
| 1164 | int d = arg_prepare_reg(gen_reg); | 1209 | orex(1,d,r,0x89); /* mov */ |
| 1165 | orex(1,d,r,0x89); /* mov */ | 1210 | o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d)); |
| 1166 | o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d)); | 1211 | if (reg_count == 2) { |
| 1167 | if (reg_count == 2) { | 1212 | /* Second word of two-word value should always be in rdx |
| 1168 | /* Second word of two-word value should always be in rdx | 1213 | this case is handled via RC_IRET */ |
| 1169 | this case is handled via RC_IRET */ | 1214 | d = arg_prepare_reg(gen_reg+1); |
| 1170 | assert(vtop->r2 == TREG_RDX); | 1215 | orex(1,d,vtop->r2,0x89); /* mov */ |
| 1171 | d = arg_prepare_reg(gen_reg+1); | 1216 | o(0xc0 + REG_VALUE(vtop->r2) * 8 + REG_VALUE(d)); |
| 1172 | orex(1,d,vtop->r2,0x89); /* mov */ | ||
| 1173 | o(0xc0 + REG_VALUE(vtop->r2) * 8 + REG_VALUE(d)); | ||
| 1174 | } | ||
| 1175 | } | 1217 | } |
| 1176 | break; | ||
| 1177 | } | 1218 | } |
| 1178 | vtop--; | 1219 | vtop--; |
| 1179 | } | 1220 | } |
| 1221 | assert(gen_reg == 0); | ||
| 1222 | assert(sse_reg == 0); | ||
| 1180 | 1223 | ||
| 1181 | /* We shouldn't have many operands on the stack anymore, but the | 1224 | /* We shouldn't have many operands on the stack anymore, but the |
| 1182 | call address itself is still there, and it might be in %eax | 1225 | call address itself is still there, and it might be in %eax |