History

Fixed x86-64 long double passing.

long double arguments require 16-byte alignment on the stack, which requires adjustment when the the stack offset is not an evven number of 8-byte words.

James Lyon committed 13 years ago

Commit 6ee366e1 parent 41d76e14 files · +206 −123Browse files

Unified · Split

4 files
  1. include/stdarg.h +2 −2
  2. lib/libtcc1.c +3 −1
  3. tests/abitest.c +19 −0
  4. x86_64-gen.c +182 −120
include/stdarg.h+2−2
@@ -7,13 +7,13 @@ · 6 unmodified lines
77 typedef void *va_list;
88
99 va_list __va_start(void *fp);
10−void *__va_arg(va_list ap, int arg_type, int size);
10+void *__va_arg(va_list ap, int arg_type, int size, int align);
1111 va_list __va_copy(va_list src);
1212 void __va_end(va_list ap);
1313
1414 #define va_start(ap, last) ((ap) = __va_start(__builtin_frame_address(0)))
1515 #define va_arg(ap, type) \
16− (*(type *)(__va_arg(ap, __builtin_va_arg_types(type), sizeof(type))))
16+ (*(type *)(__va_arg(ap, __builtin_va_arg_types(type), sizeof(type), __alignof__(type))))
1717 #define va_copy(dest, src) ((dest) = __va_copy(src))
1818 #define va_end(ap) __va_end(ap)
1919
lib/libtcc1.c+3−1
@@ -645,9 +645,10 @@ void *__va_start(void *fp) · 644 unmodified lines
645645
646646 void *__va_arg(struct __va_list_struct *ap,
647647 enum __va_arg_type arg_type,
648− int size)
648+ int size, int align)
649649 {
650650 size = (size + 7) & ~7;
651+ align = (align + 7) & ~7;
651652 switch (arg_type) {
652653 case __va_gen_reg:
653654 if (ap->gp_offset < 48) {
@@ -668,6 +669,7 @@ void *__va_arg(struct __va_list_struct *ap, · 14 unmodified lines
668669 case __va_stack:
669670 use_overflow_area:
670671 ap->overflow_arg_area += size;
672+ ap->overflow_arg_area = (char*)((long long)(ap->overflow_arg_area + align - 1) & -(long long)align);
671673 return ap->overflow_arg_area - size;
672674
673675 default:
tests/abitest.c+19−0
@@ -389,6 +389,24 @@ static int stdarg_struct_test(void) { · 388 unmodified lines
389389 return run_callback(src, stdarg_struct_test_callback);
390390 }
391391
392+/* Test that x86-64 arranges the stack correctly for arguments with alignment >8 bytes */
393+
394+typedef LONG_DOUBLE (*arg_align_test_callback_type) (LONG_DOUBLE,int,LONG_DOUBLE,int,LONG_DOUBLE);
395+
396+static int arg_align_test_callback(void *ptr) {
397+ arg_align_test_callback_type f = (arg_align_test_callback_type)ptr;
398+ long double x = f(12, 0, 25, 0, 37);
399+ return (x == 74) ? 0 : -1;
400+}
401+
402+static int arg_align_test(void) {
403+ const char *src =
404+ "long double f(long double a, int b, long double c, int d, long double e) {\n"
405+ " return a + c + e;\n"
406+ "}\n";
407+ return run_callback(src, arg_align_test_callback);
408+}
409+
392410 #define RUN_TEST(t) \
393411 if (!testname || (strcmp(#t, testname) == 0)) { \
394412 fputs(#t "... ", stdout); \
@@ -432,5 +450,6 @@ int main(int argc, char **argv) { · 37 unmodified lines
432450 RUN_TEST(many_struct_test_2);
433451 RUN_TEST(stdarg_test);
434452 RUN_TEST(stdarg_struct_test);
453+ RUN_TEST(arg_align_test);
435454 return retval;
436455 }
x86_64-gen.c+182−120
@@ -96,9 +96,9 @@ enum { · 95 unmodified lines
9696
9797 /* long double size and alignment, in bytes */
9898 #define LDOUBLE_SIZE 16
99−#define LDOUBLE_ALIGN 8
99+#define LDOUBLE_ALIGN 16
100100 /* maximum alignment (for aligned attribute support) */
101−#define MAX_ALIGN 8
101+#define MAX_ALIGN 16
102102
103103 /******************************************************/
104104 /* ELF defines */
@@ -983,7 +983,7 @@ static X86_64_Mode classify_x86_64_inner(CType *ty) { · 878 unmodified lines
983983 assert(0);
984984 }
985985
986−static X86_64_Mode classify_x86_64_arg(CType *ty, CType *ret, int *psize, int *reg_count) {
986+static X86_64_Mode classify_x86_64_arg(CType *ty, CType *ret, int *psize, int *palign, int *reg_count) {
987987 X86_64_Mode mode;
988988 int size, align, ret_t;
989989
@@ -995,6 +995,7 @@ static X86_64_Mode classify_x86_64_arg(CType *ty, CType *ret, int *psize, int *r · 5 unmodified lines
995995 } else {
996996 size = type_size(ty, &align);
997997 *psize = (size + 7) & ~7;
998+ *palign = (align + 7) & ~7;
998999
9991000 if (size > 16) {
10001001 mode = x86_64_mode_memory;
@@ -1042,8 +1043,8 @@ ST_FUNC int classify_x86_64_va_arg(CType *ty) { · 41 unmodified lines
10421043 enum __va_arg_type {
10431044 __va_gen_reg, __va_float_reg, __va_stack
10441045 };
1045− int size, reg_count;
1046− X86_64_Mode mode = classify_x86_64_arg(ty, NULL, &size, &reg_count);
1046+ int size, align, reg_count;
1047+ X86_64_Mode mode = classify_x86_64_arg(ty, NULL, &size, &align, &reg_count);
10471048 switch (mode) {
10481049 default: return __va_stack;
10491050 case x86_64_mode_integer: return __va_gen_reg;
@@ -1053,9 +1054,9 @@ ST_FUNC int classify_x86_64_va_arg(CType *ty) { · 3 unmodified lines
10531054
10541055 /* Return 1 if this function returns via an sret pointer, 0 otherwise */
10551056 int gfunc_sret(CType *vt, CType *ret, int *ret_align) {
1056− int size, reg_count;
1057+ int size, align, reg_count;
10571058 *ret_align = 1; // Never have to re-align return values for x86-64
1058− return (classify_x86_64_arg(vt, ret, &size, &reg_count) == x86_64_mode_memory);
1059+ return (classify_x86_64_arg(vt, ret, &size, &align, &reg_count) == x86_64_mode_memory);
10591060 }
10601061
10611062 #define REGN 6
@@ -1078,77 +1079,187 @@ void gfunc_call(int nb_args) · 16 unmodified lines
10781079 {
10791080 X86_64_Mode mode;
10801081 CType type;
1081− int size, align, r, args_size, i, j, reg_count;
1082+ int size, align, r, args_size, stack_adjust, run_start, run_end, i, j, reg_count;
10821083 int nb_reg_args = 0;
10831084 int nb_sse_args = 0;
10841085 int sse_reg, gen_reg;
10851086
1086− /* calculate the number of integer/float arguments */
1087− args_size = 0;
1087+ /* calculate the number of integer/float register arguments */
10881088 for(i = 0; i < nb_args; i++) {
1089− mode = classify_x86_64_arg(&vtop[-i].type, NULL, &size, &reg_count);
1090− switch (mode) {
1091− case x86_64_mode_memory:
1092− case x86_64_mode_x87:
1093− args_size += size;
1094− break;
1095−
1096− case x86_64_mode_sse:
1089+ mode = classify_x86_64_arg(&vtop[-i].type, NULL, &size, &align, &reg_count);
1090+ if (mode == x86_64_mode_sse)
10971091 nb_sse_args += reg_count;
1098− if (nb_sse_args > 8) args_size += size;
1099− break;
1100−
1101− case x86_64_mode_integer:
1092+ else if (mode == x86_64_mode_integer)
11021093 nb_reg_args += reg_count;
1103− if (nb_reg_args > REGN) args_size += size;
1104− break;
1105− }
11061094 }
11071095
1096+ /* arguments are collected in runs. Each run is a collection of 8-byte aligned arguments
1097+ and ended by a 16-byte aligned argument. This is because, from the point of view of
1098+ the callee, argument alignment is computed from the bottom up. */
11081099 /* for struct arguments, we need to call memcpy and the function
11091100 call breaks register passing arguments we are preparing.
11101101 So, we process arguments which will be passed by stack first. */
11111102 gen_reg = nb_reg_args;
11121103 sse_reg = nb_sse_args;
1113−
1114− /* adjust stack to align SSE boundary */
1115− if (args_size &= 15) {
1116− /* fetch cpu flag before the following sub will change the value */
1117− if (vtop >= vstack && (vtop->r & VT_VALMASK) == VT_CMP)
1118− gv(RC_INT);
1119−
1120− args_size = 16 - args_size;
1121− o(0x48);
1122− oad(0xec81, args_size); /* sub $xxx, %rsp */
1123− }
1124−
1125− for(i = 0; i < nb_args;) {
1126− /* Swap argument to top, it will possibly be changed here,
1127− and might use more temps. At the end of the loop we keep
1128− in on the stack and swap it back to its original position
1129− if it is a register. */
1130− SValue tmp = vtop[0];
1131− vtop[0] = vtop[-i];
1132− vtop[-i] = tmp;
1133−
1134− mode = classify_x86_64_arg(&vtop->type, NULL, &size, &reg_count);
1104+ run_start = 0;
1105+ args_size = 0;
1106+ while (run_start != nb_args) {
1107+ int run_gen_reg = gen_reg, run_sse_reg = sse_reg;
11351108
1136− int arg_stored = 1;
1137− switch (vtop->type.t & VT_BTYPE) {
1138− case VT_STRUCT:
1139− if (mode == x86_64_mode_sse) {
1140− if (sse_reg > 8)
1141− sse_reg -= reg_count;
1109+ run_end = nb_args;
1110+ stack_adjust = 0;
1111+ for(i = run_start; (i < nb_args) && (run_end == nb_args); i++) {
1112+ mode = classify_x86_64_arg(&vtop[-i].type, NULL, &size, &align, &reg_count);
1113+ switch (mode) {
1114+ case x86_64_mode_memory:
1115+ case x86_64_mode_x87:
1116+ stack_arg:
1117+ if (align == 16)
1118+ run_end = i;
11421119 else
1120+ stack_adjust += size;
1121+ break;
1122+
1123+ case x86_64_mode_sse:
1124+ sse_reg -= reg_count;
1125+ if (sse_reg + reg_count > 8) goto stack_arg;
1126+ break;
1127+
1128+ case x86_64_mode_integer:
1129+ gen_reg -= reg_count;
1130+ if (gen_reg + reg_count > REGN) goto stack_arg;
1131+ break;
1132+ }
1133+ }
1134+
1135+ gen_reg = run_gen_reg;
1136+ sse_reg = run_sse_reg;
1137+
1138+ /* adjust stack to align SSE boundary */
1139+ if (stack_adjust &= 15) {
1140+ /* fetch cpu flag before the following sub will change the value */
1141+ if (vtop >= vstack && (vtop->r & VT_VALMASK) == VT_CMP)
1142+ gv(RC_INT);
1143+
1144+ stack_adjust = 16 - stack_adjust;
1145+ o(0x48);
1146+ oad(0xec81, stack_adjust); /* sub $xxx, %rsp */
1147+ args_size += stack_adjust;
1148+ }
1149+
1150+ for(i = run_start; i < run_end;) {
1151+ /* Swap argument to top, it will possibly be changed here,
1152+ and might use more temps. At the end of the loop we keep
1153+ in on the stack and swap it back to its original position
1154+ if it is a register. */
1155+ SValue tmp = vtop[0];
1156+ vtop[0] = vtop[-i];
1157+ vtop[-i] = tmp;
1158+
1159+ mode = classify_x86_64_arg(&vtop->type, NULL, &size, &align, &reg_count);
1160+
1161+ int arg_stored = 1;
1162+ switch (vtop->type.t & VT_BTYPE) {
1163+ case VT_STRUCT:
1164+ if (mode == x86_64_mode_sse) {
1165+ if (sse_reg > 8)
1166+ sse_reg -= reg_count;
1167+ else
1168+ arg_stored = 0;
1169+ } else if (mode == x86_64_mode_integer) {
1170+ if (gen_reg > REGN)
1171+ gen_reg -= reg_count;
1172+ else
1173+ arg_stored = 0;
1174+ }
1175+
1176+ if (arg_stored) {
1177+ /* allocate the necessary size on stack */
1178+ o(0x48);
1179+ oad(0xec81, size); /* sub $xxx, %rsp */
1180+ /* generate structure store */
1181+ r = get_reg(RC_INT);
1182+ orex(1, r, 0, 0x89); /* mov %rsp, r */
1183+ o(0xe0 + REG_VALUE(r));
1184+ vset(&vtop->type, r | VT_LVAL, 0);
1185+ vswap();
1186+ vstore();
1187+ args_size += size;
1188+ }
1189+ break;
1190+
1191+ case VT_LDOUBLE:
1192+ assert(0);
1193+ break;
1194+
1195+ case VT_FLOAT:
1196+ case VT_DOUBLE:
1197+ assert(mode == x86_64_mode_sse);
1198+ if (sse_reg > 8) {
1199+ --sse_reg;
1200+ r = gv(RC_FLOAT);
1201+ o(0x50); /* push $rax */
1202+ /* movq %xmmN, (%rsp) */
1203+ o(0xd60f66);
1204+ o(0x04 + REG_VALUE(r)*8);
1205+ o(0x24);
1206+ args_size += size;
1207+ } else {
11431208 arg_stored = 0;
1144− } else if (mode == x86_64_mode_integer) {
1145− if (gen_reg > REGN)
1146− gen_reg -= reg_count;
1147− else
1209+ }
1210+ break;
1211+
1212+ default:
1213+ assert(mode == x86_64_mode_integer);
1214+ /* simple type */
1215+ /* XXX: implicit cast ? */
1216+ if (gen_reg > REGN) {
1217+ --gen_reg;
1218+ r = gv(RC_INT);
1219+ orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */
1220+ args_size += size;
1221+ } else {
11481222 arg_stored = 0;
1223+ }
1224+ break;
11491225 }
11501226
1227+ /* And swap the argument back to it's original position. */
1228+ tmp = vtop[0];
1229+ vtop[0] = vtop[-i];
1230+ vtop[-i] = tmp;
1231+
11511232 if (arg_stored) {
1233+ vrotb(i+1);
1234+ assert((vtop->type.t == tmp.type.t) && (vtop->r == tmp.r));
1235+ vpop();
1236+ --nb_args;
1237+ --run_end;
1238+ } else {
1239+ ++i;
1240+ }
1241+ }
1242+
1243+ /* handle 16 byte aligned arguments at end of run */
1244+ run_start = i = run_end;
1245+ while (i < nb_args) {
1246+ /* Rotate argument to top since it will always be popped */
1247+ mode = classify_x86_64_arg(&vtop[-i].type, NULL, &size, &align, &reg_count);
1248+ if (align != 16)
1249+ break;
1250+
1251+ vrotb(i+1);
1252+
1253+ if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) {
1254+ gv(RC_ST0);
1255+ oad(0xec8148, size); /* sub $xxx, %rsp */
1256+ o(0x7cdb); /* fstpt 0(%rsp) */
1257+ g(0x24);
1258+ g(0x00);
1259+ args_size += size;
1260+ } else {
1261+ assert(mode == x86_64_mode_memory);
1262+
11521263 /* allocate the necessary size on stack */
11531264 o(0x48);
11541265 oad(0xec81, size); /* sub $xxx, %rsp */
@@ -1161,65 +1272,12 @@ void gfunc_call(int nb_args) · 6 unmodified lines
11611272 vstore();
11621273 args_size += size;
11631274 }
1164− break;
11651275
1166− case VT_LDOUBLE:
1167− gv(RC_ST0);
1168− size = LDOUBLE_SIZE;
1169− oad(0xec8148, size); /* sub $xxx, %rsp */
1170− o(0x7cdb); /* fstpt 0(%rsp) */
1171− g(0x24);
1172− g(0x00);
1173− args_size += size;
1174− break;
1175−
1176− case VT_FLOAT:
1177− case VT_DOUBLE:
1178− assert(mode == x86_64_mode_sse);
1179− if (sse_reg > 8) {
1180− --sse_reg;
1181− r = gv(RC_FLOAT);
1182− o(0x50); /* push $rax */
1183− /* movq %xmmN, (%rsp) */
1184− o(0xd60f66);
1185− o(0x04 + REG_VALUE(r)*8);
1186− o(0x24);
1187− args_size += size;
1188− } else {
1189− arg_stored = 0;
1190− }
1191− break;
1192−
1193− default:
1194− assert(mode == x86_64_mode_integer);
1195− /* simple type */
1196− /* XXX: implicit cast ? */
1197− if (gen_reg > REGN) {
1198− --gen_reg;
1199− r = gv(RC_INT);
1200− orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */
1201− args_size += size;
1202− } else {
1203− arg_stored = 0;
1204− }
1205− break;
1206− }
1207−
1208− /* And swap the argument back to it's original position. */
1209− tmp = vtop[0];
1210− vtop[0] = vtop[-i];
1211− vtop[-i] = tmp;
1212−
1213− if (arg_stored) {
1214− vrotb(i+1);
1215− assert(vtop->type.t == tmp.type.t);
1216− vpop();
1217− --nb_args;
1218− } else {
1219− ++i;
1276+ vpop();
1277+ --nb_args;
12201278 }
12211279 }
1222−
1280+
12231281 /* XXX This should be superfluous. */
12241282 save_regs(0); /* save used temporary registers */
12251283
@@ -1230,7 +1288,7 @@ void gfunc_call(int nb_args) · 4 unmodified lines
12301288 assert(gen_reg <= REGN);
12311289 assert(sse_reg <= 8);
12321290 for(i = 0; i < nb_args; i++) {
1233− mode = classify_x86_64_arg(&vtop->type, &type, &size, &reg_count);
1291+ mode = classify_x86_64_arg(&vtop->type, &type, &size, &align, &reg_count);
12341292 /* Alter stack entry type so that gv() knows how to treat it */
12351293 vtop->type = type;
12361294 if (mode == x86_64_mode_sse) {
@@ -1324,10 +1382,11 @@ void gfunc_prolog(CType *func_type) · 87 unmodified lines
13241382 sym = func_type->ref;
13251383 while ((sym = sym->next) != NULL) {
13261384 type = &sym->type;
1327− mode = classify_x86_64_arg(type, NULL, &size, &reg_count);
1385+ mode = classify_x86_64_arg(type, NULL, &size, &align, &reg_count);
13281386 switch (mode) {
13291387 default:
1330− seen_stack_size += size;
1388+ stack_arg:
1389+ seen_stack_size = ((seen_stack_size + align - 1) & -align) + size;
13311390 break;
13321391
13331392 case x86_64_mode_integer:
@@ -1335,7 +1394,7 @@ void gfunc_prolog(CType *func_type) · 1 unmodified line
13351394 seen_reg_num += reg_count;
13361395 } else {
13371396 seen_reg_num = 8;
1338− seen_stack_size += size;
1397+ goto stack_arg;
13391398 }
13401399 break;
13411400
@@ -1344,7 +1403,7 @@ void gfunc_prolog(CType *func_type) · 2 unmodified lines
13441403 seen_sse_num += reg_count;
13451404 } else {
13461405 seen_sse_num = 8;
1347− seen_stack_size += size;
1406+ goto stack_arg;
13481407 }
13491408 break;
13501409 }
@@ -1383,7 +1442,7 @@ void gfunc_prolog(CType *func_type) · 32 unmodified lines
13831442 /* if the function returns a structure, then add an
13841443 implicit pointer parameter */
13851444 func_vt = sym->type;
1386− mode = classify_x86_64_arg(&func_vt, NULL, &size, &reg_count);
1445+ mode = classify_x86_64_arg(&func_vt, NULL, &size, &align, &reg_count);
13871446 if (mode == x86_64_mode_memory) {
13881447 push_arg_reg(reg_param_index);
13891448 func_vc = loc;
@@ -1392,7 +1451,7 @@ void gfunc_prolog(CType *func_type) · 2 unmodified lines
13921451 /* define parameters */
13931452 while ((sym = sym->next) != NULL) {
13941453 type = &sym->type;
1395− mode = classify_x86_64_arg(type, NULL, &size, &reg_count);
1454+ mode = classify_x86_64_arg(type, NULL, &size, &align, &reg_count);
13961455 switch (mode) {
13971456 case x86_64_mode_sse:
13981457 if (sse_param_index + reg_count <= 8) {
@@ -1405,6 +1464,7 @@ void gfunc_prolog(CType *func_type) · 6 unmodified lines
14051464 ++sse_param_index;
14061465 }
14071466 } else {
1467+ addr = (addr + align - 1) & -align;
14081468 param_addr = addr;
14091469 addr += size;
14101470 sse_param_index += reg_count;
@@ -1413,6 +1473,7 @@ void gfunc_prolog(CType *func_type) · 2 unmodified lines
14131473
14141474 case x86_64_mode_memory:
14151475 case x86_64_mode_x87:
1476+ addr = (addr + align - 1) & -align;
14161477 param_addr = addr;
14171478 addr += size;
14181479 break;
@@ -1427,6 +1488,7 @@ void gfunc_prolog(CType *func_type) · 8 unmodified lines
14271488 ++reg_param_index;
14281489 }
14291490 } else {
1491+ addr = (addr + align - 1) & -align;
14301492 param_addr = addr;
14311493 addr += size;
14321494 reg_param_index += reg_count;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t