| @@ -602,6 +602,12 @@ void gen_offs_sp(int b, int r, int d) · 601 unmodified lines |
| 602 | 602 | } |
| 603 | 603 | } |
| 604 | 604 | |
| 605 | +/* Return 1 if this function returns via an sret pointer, 0 otherwise */ |
| 606 | +ST_FUNC int gfunc_sret(CType *vt, CType *ret, int *ret_align) { |
| 607 | + *ret_align = 1; // Never have to re-align return values for x86-64 |
| 608 | + return 1; |
| 609 | +} |
| 610 | + |
| 605 | 611 | void gfunc_call(int nb_args) |
| 606 | 612 | { |
| 607 | 613 | int size, align, r, args_size, i, d, j, bt, struct_size; |
| @@ -817,6 +823,139 @@ static void gadd_sp(int val) · 209 unmodified lines |
| 817 | 823 | } |
| 818 | 824 | } |
| 819 | 825 | |
| 826 | +typedef enum X86_64_Mode { |
| 827 | + x86_64_mode_none, |
| 828 | + x86_64_mode_memory, |
| 829 | + x86_64_mode_integer, |
| 830 | + x86_64_mode_sse, |
| 831 | + x86_64_mode_x87 |
| 832 | +} X86_64_Mode; |
| 833 | + |
| 834 | +static X86_64_Mode classify_x86_64_merge(X86_64_Mode a, X86_64_Mode b) { |
| 835 | + if (a == b) |
| 836 | + return a; |
| 837 | + else if (a == x86_64_mode_none) |
| 838 | + return b; |
| 839 | + else if (b == x86_64_mode_none) |
| 840 | + return a; |
| 841 | + else if ((a == x86_64_mode_memory) || (b == x86_64_mode_memory)) |
| 842 | + return x86_64_mode_memory; |
| 843 | + else if ((a == x86_64_mode_integer) || (b == x86_64_mode_integer)) |
| 844 | + return x86_64_mode_integer; |
| 845 | + else if ((a == x86_64_mode_x87) || (b == x86_64_mode_x87)) |
| 846 | + return x86_64_mode_memory; |
| 847 | + else |
| 848 | + return x86_64_mode_sse; |
| 849 | +} |
| 850 | + |
| 851 | +static X86_64_Mode classify_x86_64_inner(CType *ty) { |
| 852 | + X86_64_Mode mode; |
| 853 | + Sym *f; |
| 854 | + |
| 855 | + if (ty->t & VT_BITFIELD) |
| 856 | + return x86_64_mode_memory; |
| 857 | + |
| 858 | + switch (ty->t & VT_BTYPE) { |
| 859 | + case VT_VOID: return x86_64_mode_none; |
| 860 | + |
| 861 | + case VT_INT: |
| 862 | + case VT_BYTE: |
| 863 | + case VT_SHORT: |
| 864 | + case VT_LLONG: |
| 865 | + case VT_BOOL: |
| 866 | + case VT_PTR: |
| 867 | + case VT_ENUM: return x86_64_mode_integer; |
| 868 | + |
| 869 | + case VT_FLOAT: |
| 870 | + case VT_DOUBLE: return x86_64_mode_sse; |
| 871 | + |
| 872 | + case VT_LDOUBLE: return x86_64_mode_x87; |
| 873 | + |
| 874 | + case VT_STRUCT: |
| 875 | + f = ty->ref; |
| 876 | + |
| 877 | + // Detect union |
| 878 | + if (f->next && (f->c == f->next->c)) |
| 879 | + return x86_64_mode_memory; |
| 880 | + |
| 881 | + mode = x86_64_mode_none; |
| 882 | + for (; f; f = f->next) |
| 883 | + mode = classify_x86_64_merge(mode, classify_x86_64_inner(&f->type)); |
| 884 | + |
| 885 | + return mode; |
| 886 | + } |
| 887 | +} |
| 888 | + |
| 889 | +static X86_64_Mode classify_x86_64_arg(CType *ty, int *psize, int *reg_count) { |
| 890 | + X86_64_Mode mode; |
| 891 | + int size, align; |
| 892 | + |
| 893 | + if (ty->t & VT_ARRAY) { |
| 894 | + *psize = 8; |
| 895 | + *reg_count = 1; |
| 896 | + return x86_64_mode_integer; |
| 897 | + } |
| 898 | + |
| 899 | + size = type_size(ty, &align); |
| 900 | + size = (size + 7) & ~7; |
| 901 | + *psize = size; |
| 902 | + if (size > 16) |
| 903 | + return x86_64_mode_memory; |
| 904 | + |
| 905 | + mode = classify_x86_64_inner(ty); |
| 906 | + if (reg_count) { |
| 907 | + if (mode == x86_64_mode_integer) |
| 908 | + *reg_count = size / 8; |
| 909 | + else if (mode == x86_64_mode_none) |
| 910 | + *reg_count = 0; |
| 911 | + else |
| 912 | + *reg_count = 1; |
| 913 | + } |
| 914 | + return mode; |
| 915 | +} |
| 916 | + |
| 917 | +static X86_64_Mode classify_x86_64_arg_type(CType *vt, CType *ret, int *psize, int *reg_count) { |
| 918 | + X86_64_Mode mode; |
| 919 | + int size; |
| 920 | + |
| 921 | + ret->ref = NULL; |
| 922 | + |
| 923 | + mode = classify_x86_64_arg(vt, &size, reg_count); |
| 924 | + *psize = size; |
| 925 | + switch (mode) { |
| 926 | + case x86_64_mode_integer: |
| 927 | + if (size > 8) |
| 928 | + ret->t = VT_QLONG; |
| 929 | + else if (size > 4) |
| 930 | + ret->t = VT_LLONG; |
| 931 | + else |
| 932 | + ret->t = VT_INT; |
| 933 | + break; |
| 934 | + |
| 935 | + case x86_64_mode_x87: |
| 936 | + ret->t = VT_LDOUBLE; |
| 937 | + break; |
| 938 | + |
| 939 | + case x86_64_mode_sse: |
| 940 | + if (size > 8) |
| 941 | + ret->t = VT_QFLOAT; |
| 942 | + else if (size > 4) |
| 943 | + ret->t = VT_DOUBLE; |
| 944 | + else |
| 945 | + ret->t = VT_FLOAT; |
| 946 | + break; |
| 947 | + } |
| 948 | + |
| 949 | + return mode; |
| 950 | +} |
| 951 | + |
| 952 | +/* Return 1 if this function returns via an sret pointer, 0 otherwise */ |
| 953 | +int gfunc_sret(CType *vt, CType *ret, int *ret_align) { |
| 954 | + int size, reg_count; |
| 955 | + *ret_align = 1; // Never have to re-align return values for x86-64 |
| 956 | + return (classify_x86_64_arg_type(vt, ret, &size, ®_count) == x86_64_mode_memory); |
| 957 | +} |
| 958 | + |
| 820 | 959 | #define REGN 6 |
| 821 | 960 | static const uint8_t arg_regs[REGN] = { |
| 822 | 961 | TREG_RDI, TREG_RSI, TREG_RDX, TREG_RCX, TREG_R8, TREG_R9 |
| @@ -827,7 +966,9 @@ static const uint8_t arg_regs[REGN] = { · 4 unmodified lines |
| 827 | 966 | parameters and the function address. */ |
| 828 | 967 | void gfunc_call(int nb_args) |
| 829 | 968 | { |
| 830 | | − int size, align, r, args_size, i; |
| 969 | + X86_64_Mode mode; |
| 970 | + CType type; |
| 971 | + int size, align, r, args_size, i, j, reg_count; |
| 831 | 972 | int nb_reg_args = 0; |
| 832 | 973 | int nb_sse_args = 0; |
| 833 | 974 | int sse_reg, gen_reg; |
| @@ -835,17 +976,22 @@ void gfunc_call(int nb_args) · 1 unmodified line |
| 835 | 976 | /* calculate the number of integer/float arguments */ |
| 836 | 977 | args_size = 0; |
| 837 | 978 | for(i = 0; i < nb_args; i++) { |
| 838 | | − if ((vtop[-i].type.t & VT_BTYPE) == VT_STRUCT) { |
| 839 | | − args_size += type_size(&vtop[-i].type, &align); |
| 840 | | − args_size = (args_size + 7) & ~7; |
| 841 | | − } else if ((vtop[-i].type.t & VT_BTYPE) == VT_LDOUBLE) { |
| 842 | | − args_size += 16; |
| 843 | | − } else if (is_sse_float(vtop[-i].type.t)) { |
| 844 | | − nb_sse_args++; |
| 845 | | − if (nb_sse_args > 8) args_size += 8; |
| 846 | | − } else { |
| 847 | | − nb_reg_args++; |
| 848 | | − if (nb_reg_args > REGN) args_size += 8; |
| 979 | + mode = classify_x86_64_arg(&vtop[-i].type, &size, ®_count); |
| 980 | + switch (mode) { |
| 981 | + case x86_64_mode_memory: |
| 982 | + case x86_64_mode_x87: |
| 983 | + args_size += size; |
| 984 | + break; |
| 985 | + |
| 986 | + case x86_64_mode_sse: |
| 987 | + nb_sse_args += reg_count; |
| 988 | + if (nb_sse_args > 8) args_size += size; |
| 989 | + break; |
| 990 | + |
| 991 | + case x86_64_mode_integer: |
| 992 | + nb_reg_args += reg_count; |
| 993 | + if (nb_reg_args > REGN) args_size += size; |
| 994 | + break; |
| 849 | 995 | } |
| 850 | 996 | } |
| 851 | 997 | |
| @@ -875,10 +1021,9 @@ void gfunc_call(int nb_args) · 23 unmodified lines |
| 875 | 1021 | SValue tmp = vtop[0]; |
| 876 | 1022 | vtop[0] = vtop[-i]; |
| 877 | 1023 | vtop[-i] = tmp; |
| 878 | | − if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) { |
| 879 | | − size = type_size(&vtop->type, &align); |
| 880 | | − /* align to stack align size */ |
| 881 | | − size = (size + 7) & ~7; |
| 1024 | + mode = classify_x86_64_arg(&vtop->type, &size, ®_count); |
| 1025 | + switch (mode) { |
| 1026 | + case x86_64_mode_memory: |
| 882 | 1027 | /* allocate the necessary size on stack */ |
| 883 | 1028 | o(0x48); |
| 884 | 1029 | oad(0xec81, size); /* sub $xxx, %rsp */ |
| @@ -890,7 +1035,9 @@ void gfunc_call(int nb_args) · 5 unmodified lines |
| 890 | 1035 | vswap(); |
| 891 | 1036 | vstore(); |
| 892 | 1037 | args_size += size; |
| 893 | | − } else if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) { |
| 1038 | + break; |
| 1039 | + |
| 1040 | + case x86_64_mode_x87: |
| 894 | 1041 | gv(RC_ST0); |
| 895 | 1042 | size = LDOUBLE_SIZE; |
| 896 | 1043 | oad(0xec8148, size); /* sub $xxx, %rsp */ |
| @@ -898,25 +1045,30 @@ void gfunc_call(int nb_args) · 1 unmodified line |
| 898 | 1045 | g(0x24); |
| 899 | 1046 | g(0x00); |
| 900 | 1047 | args_size += size; |
| 901 | | − } else if (is_sse_float(vtop->type.t)) { |
| 902 | | − int j = --sse_reg; |
| 903 | | − if (j >= 8) { |
| 1048 | + break; |
| 1049 | + |
| 1050 | + case x86_64_mode_sse: |
| 1051 | + if (sse_reg > 8) { |
| 904 | 1052 | gv(RC_FLOAT); |
| 905 | 1053 | o(0x50); /* push $rax */ |
| 906 | 1054 | /* movq %xmm0, (%rsp) */ |
| 907 | 1055 | o(0x04d60f66); |
| 908 | 1056 | o(0x24); |
| 909 | | − args_size += 8; |
| 1057 | + args_size += size; |
| 910 | 1058 | } |
| 911 | | − } else { |
| 912 | | − int j = --gen_reg; |
| 1059 | + sse_reg -= reg_count; |
| 1060 | + break; |
| 1061 | + |
| 1062 | + case x86_64_mode_integer: |
| 913 | 1063 | /* simple type */ |
| 914 | 1064 | /* XXX: implicit cast ? */ |
| 915 | | − if (j >= REGN) { |
| 1065 | + if (gen_reg > REGN) { |
| 916 | 1066 | r = gv(RC_INT); |
| 917 | 1067 | orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */ |
| 918 | | − args_size += 8; |
| 1068 | + args_size += size; |
| 919 | 1069 | } |
| 1070 | + gen_reg -= reg_count; |
| 1071 | + break; |
| 920 | 1072 | } |
| 921 | 1073 | |
| 922 | 1074 | /* And swap the argument back to it's original position. */ |
| @@ -935,29 +1087,45 @@ void gfunc_call(int nb_args) · 12 unmodified lines |
| 935 | 1087 | gen_reg = nb_reg_args; |
| 936 | 1088 | sse_reg = nb_sse_args; |
| 937 | 1089 | for(i = 0; i < nb_args; i++) { |
| 938 | | − if ((vtop->type.t & VT_BTYPE) == VT_STRUCT || |
| 939 | | − (vtop->type.t & VT_BTYPE) == VT_LDOUBLE) { |
| 940 | | − } else if (is_sse_float(vtop->type.t)) { |
| 941 | | − int j = --sse_reg; |
| 942 | | − if (j < 8) { |
| 943 | | − gv(RC_FLOAT); /* only one float register */ |
| 944 | | − /* movaps %xmm0, %xmmN */ |
| 945 | | − o(0x280f); |
| 946 | | − o(0xc0 + (sse_reg << 3)); |
| 1090 | + mode = classify_x86_64_arg_type(&vtop->type, &type, &size, ®_count); |
| 1091 | + /* Alter stack entry type so that gv() knows how to treat it */ |
| 1092 | + vtop->type = type; |
| 1093 | + switch (mode) { |
| 1094 | + default: |
| 1095 | + break; |
| 1096 | + |
| 1097 | + case x86_64_mode_sse: |
| 1098 | + if (sse_reg > 8) { |
| 1099 | + sse_reg -= reg_count; |
| 1100 | + } else { |
| 1101 | + for (j = 0; j < reg_count; ++j) { |
| 1102 | + --sse_reg; |
| 1103 | + gv(RC_FLOAT); /* only one float register */ |
| 1104 | + /* movaps %xmm0, %xmmN */ |
| 1105 | + o(0x280f); |
| 1106 | + o(0xc0 + (sse_reg << 3)); |
| 1107 | + } |
| 947 | 1108 | } |
| 948 | | − } else { |
| 949 | | − int j = --gen_reg; |
| 1109 | + break; |
| 1110 | + |
| 1111 | + case x86_64_mode_integer: |
| 950 | 1112 | /* simple type */ |
| 951 | 1113 | /* XXX: implicit cast ? */ |
| 952 | | − if (j < REGN) { |
| 953 | | − int d = arg_regs[j]; |
| 954 | | − r = gv(RC_INT); |
| 955 | | − if (j == 2 || j == 3) |
| 956 | | − /* j=2: r10, j=3: r11 */ |
| 957 | | − d = j + 8; |
| 958 | | − orex(1,d,r,0x89); /* mov */ |
| 959 | | − o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d)); |
| 1114 | + if (gen_reg > 8) { |
| 1115 | + gen_reg -= reg_count; |
| 1116 | + } else { |
| 1117 | + for (j = 0; j < reg_count; ++j) { |
| 1118 | + --gen_reg; |
| 1119 | + int d = arg_regs[gen_reg]; |
| 1120 | + r = gv(RC_INT); |
| 1121 | + if (gen_reg == 2 || gen_reg == 3) |
| 1122 | + /* gen_reg=2: r10, gen_reg=3: r11 */ |
| 1123 | + d = gen_reg + 8; |
| 1124 | + orex(1,d,r,0x89); /* mov */ |
| 1125 | + o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d)); |
| 1126 | + } |
| 960 | 1127 | } |
| 1128 | + break; |
| 961 | 1129 | } |
| 962 | 1130 | vtop--; |
| 963 | 1131 | } |
| @@ -994,7 +1162,8 @@ static void push_arg_reg(int i) { · 30 unmodified lines |
| 994 | 1162 | /* generate function prolog of type 't' */ |
| 995 | 1163 | void gfunc_prolog(CType *func_type) |
| 996 | 1164 | { |
| 997 | | − int i, addr, align, size; |
| 1165 | + X86_64_Mode mode; |
| 1166 | + int i, addr, align, size, reg_count; |
| 998 | 1167 | int param_index, param_addr, reg_param_index, sse_param_index; |
| 999 | 1168 | Sym *sym; |
| 1000 | 1169 | CType *type; |
| @@ -1070,7 +1239,8 @@ void gfunc_prolog(CType *func_type) · 69 unmodified lines |
| 1070 | 1239 | /* if the function returns a structure, then add an |
| 1071 | 1240 | implicit pointer parameter */ |
| 1072 | 1241 | func_vt = sym->type; |
| 1073 | | − if ((func_vt.t & VT_BTYPE) == VT_STRUCT) { |
| 1242 | + mode = classify_x86_64_arg(&func_vt, &size, ®_count); |
| 1243 | + if (mode == x86_64_mode_memory) { |
| 1074 | 1244 | push_arg_reg(reg_param_index); |
| 1075 | 1245 | param_addr = loc; |
| 1076 | 1246 | |
| @@ -1081,35 +1251,46 @@ void gfunc_prolog(CType *func_type) · 4 unmodified lines |
| 1081 | 1251 | /* define parameters */ |
| 1082 | 1252 | while ((sym = sym->next) != NULL) { |
| 1083 | 1253 | type = &sym->type; |
| 1084 | | − size = type_size(type, &align); |
| 1085 | | − size = (size + 7) & ~7; |
| 1086 | | − if (is_sse_float(type->t)) { |
| 1087 | | − if (sse_param_index < 8) { |
| 1254 | + mode = classify_x86_64_arg(type, &size, ®_count); |
| 1255 | + switch (mode) { |
| 1256 | + case x86_64_mode_sse: |
| 1257 | + if (sse_param_index + reg_count <= 8) { |
| 1088 | 1258 | /* save arguments passed by register */ |
| 1089 | | − loc -= 8; |
| 1090 | | − o(0xd60f66); /* movq */ |
| 1091 | | − gen_modrm(sse_param_index, VT_LOCAL, NULL, loc); |
| 1259 | + for (i = 0; i < reg_count; ++i) { |
| 1260 | + loc -= 8; |
| 1261 | + o(0xd60f66); /* movq */ |
| 1262 | + gen_modrm(sse_param_index, VT_LOCAL, NULL, loc); |
| 1263 | + ++sse_param_index; |
| 1264 | + } |
| 1092 | 1265 | param_addr = loc; |
| 1093 | 1266 | } else { |
| 1094 | 1267 | param_addr = addr; |
| 1095 | 1268 | addr += size; |
| 1269 | + sse_param_index += reg_count; |
| 1096 | 1270 | } |
| 1097 | | − sse_param_index++; |
| 1098 | | − |
| 1099 | | − } else if ((type->t & VT_BTYPE) == VT_STRUCT || |
| 1100 | | − (type->t & VT_BTYPE) == VT_LDOUBLE) { |
| 1271 | + break; |
| 1272 | + |
| 1273 | + case x86_64_mode_memory: |
| 1274 | + case x86_64_mode_x87: |
| 1101 | 1275 | param_addr = addr; |
| 1102 | 1276 | addr += size; |
| 1103 | | − } else { |
| 1104 | | − if (reg_param_index < REGN) { |
| 1277 | + break; |
| 1278 | + |
| 1279 | + case x86_64_mode_integer: { |
| 1280 | + if (reg_param_index + reg_count <= REGN) { |
| 1105 | 1281 | /* save arguments passed by register */ |
| 1106 | | − push_arg_reg(reg_param_index); |
| 1282 | + for (i = 0; i < reg_count; ++i) { |
| 1283 | + push_arg_reg(reg_param_index); |
| 1284 | + ++reg_param_index; |
| 1285 | + } |
| 1107 | 1286 | param_addr = loc; |
| 1108 | 1287 | } else { |
| 1109 | 1288 | param_addr = addr; |
| 1110 | | − addr += 8; |
| 1289 | + addr += size; |
| 1290 | + reg_param_index += reg_count; |
| 1111 | 1291 | } |
| 1112 | | − reg_param_index++; |
| 1292 | + break; |
| 1293 | + } |
| 1113 | 1294 | } |
| 1114 | 1295 | sym_push(sym->v & ~SYM_FIELD, type, |
| 1115 | 1296 | VT_LOCAL | VT_LVAL, param_addr); |