| @@ -602,6 +602,12 @@ void gen_offs_sp(int b, int r, int d) · 601 unmodified lines |
| 602 | } | 602 | } |
| 603 | } | 603 | } |
| 604 | | 604 | |
| | 605 | /* Return 1 if this function returns via an sret pointer, 0 otherwise */ |
| | 606 | ST_FUNC int gfunc_sret(CType *vt, CType *ret, int *ret_align) { |
| | 607 | *ret_align = 1; // Never have to re-align return values for x86-64 |
| | 608 | return 1; |
| | 609 | } |
| | 610 | |
| 605 | void gfunc_call(int nb_args) | 611 | void gfunc_call(int nb_args) |
| 606 | { | 612 | { |
| 607 | int size, align, r, args_size, i, d, j, bt, struct_size; | 613 | int size, align, r, args_size, i, d, j, bt, struct_size; |
| @@ -817,6 +823,139 @@ static void gadd_sp(int val) · 209 unmodified lines |
| 817 | } | 823 | } |
| 818 | } | 824 | } |
| 819 | | 825 | |
| | 826 | typedef enum X86_64_Mode { |
| | 827 | x86_64_mode_none, |
| | 828 | x86_64_mode_memory, |
| | 829 | x86_64_mode_integer, |
| | 830 | x86_64_mode_sse, |
| | 831 | x86_64_mode_x87 |
| | 832 | } X86_64_Mode; |
| | 833 | |
| | 834 | static X86_64_Mode classify_x86_64_merge(X86_64_Mode a, X86_64_Mode b) { |
| | 835 | if (a == b) |
| | 836 | return a; |
| | 837 | else if (a == x86_64_mode_none) |
| | 838 | return b; |
| | 839 | else if (b == x86_64_mode_none) |
| | 840 | return a; |
| | 841 | else if ((a == x86_64_mode_memory) || (b == x86_64_mode_memory)) |
| | 842 | return x86_64_mode_memory; |
| | 843 | else if ((a == x86_64_mode_integer) || (b == x86_64_mode_integer)) |
| | 844 | return x86_64_mode_integer; |
| | 845 | else if ((a == x86_64_mode_x87) || (b == x86_64_mode_x87)) |
| | 846 | return x86_64_mode_memory; |
| | 847 | else |
| | 848 | return x86_64_mode_sse; |
| | 849 | } |
| | 850 | |
| | 851 | static X86_64_Mode classify_x86_64_inner(CType *ty) { |
| | 852 | X86_64_Mode mode; |
| | 853 | Sym *f; |
| | 854 | |
| | 855 | if (ty->t & VT_BITFIELD) |
| | 856 | return x86_64_mode_memory; |
| | 857 | |
| | 858 | switch (ty->t & VT_BTYPE) { |
| | 859 | case VT_VOID: return x86_64_mode_none; |
| | 860 | |
| | 861 | case VT_INT: |
| | 862 | case VT_BYTE: |
| | 863 | case VT_SHORT: |
| | 864 | case VT_LLONG: |
| | 865 | case VT_BOOL: |
| | 866 | case VT_PTR: |
| | 867 | case VT_ENUM: return x86_64_mode_integer; |
| | 868 | |
| | 869 | case VT_FLOAT: |
| | 870 | case VT_DOUBLE: return x86_64_mode_sse; |
| | 871 | |
| | 872 | case VT_LDOUBLE: return x86_64_mode_x87; |
| | 873 | |
| | 874 | case VT_STRUCT: |
| | 875 | f = ty->ref; |
| | 876 | |
| | 877 | // Detect union |
| | 878 | if (f->next && (f->c == f->next->c)) |
| | 879 | return x86_64_mode_memory; |
| | 880 | |
| | 881 | mode = x86_64_mode_none; |
| | 882 | for (; f; f = f->next) |
| | 883 | mode = classify_x86_64_merge(mode, classify_x86_64_inner(&f->type)); |
| | 884 | |
| | 885 | return mode; |
| | 886 | } |
| | 887 | } |
| | 888 | |
| | 889 | static X86_64_Mode classify_x86_64_arg(CType *ty, int *psize, int *reg_count) { |
| | 890 | X86_64_Mode mode; |
| | 891 | int size, align; |
| | 892 | |
| | 893 | if (ty->t & VT_ARRAY) { |
| | 894 | *psize = 8; |
| | 895 | *reg_count = 1; |
| | 896 | return x86_64_mode_integer; |
| | 897 | } |
| | 898 | |
| | 899 | size = type_size(ty, &align); |
| | 900 | size = (size + 7) & ~7; |
| | 901 | *psize = size; |
| | 902 | if (size > 16) |
| | 903 | return x86_64_mode_memory; |
| | 904 | |
| | 905 | mode = classify_x86_64_inner(ty); |
| | 906 | if (reg_count) { |
| | 907 | if (mode == x86_64_mode_integer) |
| | 908 | *reg_count = size / 8; |
| | 909 | else if (mode == x86_64_mode_none) |
| | 910 | *reg_count = 0; |
| | 911 | else |
| | 912 | *reg_count = 1; |
| | 913 | } |
| | 914 | return mode; |
| | 915 | } |
| | 916 | |
| | 917 | static X86_64_Mode classify_x86_64_arg_type(CType *vt, CType *ret, int *psize, int *reg_count) { |
| | 918 | X86_64_Mode mode; |
| | 919 | int size; |
| | 920 | |
| | 921 | ret->ref = NULL; |
| | 922 | |
| | 923 | mode = classify_x86_64_arg(vt, &size, reg_count); |
| | 924 | *psize = size; |
| | 925 | switch (mode) { |
| | 926 | case x86_64_mode_integer: |
| | 927 | if (size > 8) |
| | 928 | ret->t = VT_QLONG; |
| | 929 | else if (size > 4) |
| | 930 | ret->t = VT_LLONG; |
| | 931 | else |
| | 932 | ret->t = VT_INT; |
| | 933 | break; |
| | 934 | |
| | 935 | case x86_64_mode_x87: |
| | 936 | ret->t = VT_LDOUBLE; |
| | 937 | break; |
| | 938 | |
| | 939 | case x86_64_mode_sse: |
| | 940 | if (size > 8) |
| | 941 | ret->t = VT_QFLOAT; |
| | 942 | else if (size > 4) |
| | 943 | ret->t = VT_DOUBLE; |
| | 944 | else |
| | 945 | ret->t = VT_FLOAT; |
| | 946 | break; |
| | 947 | } |
| | 948 | |
| | 949 | return mode; |
| | 950 | } |
| | 951 | |
| | 952 | /* Return 1 if this function returns via an sret pointer, 0 otherwise */ |
| | 953 | int gfunc_sret(CType *vt, CType *ret, int *ret_align) { |
| | 954 | int size, reg_count; |
| | 955 | *ret_align = 1; // Never have to re-align return values for x86-64 |
| | 956 | return (classify_x86_64_arg_type(vt, ret, &size, ®_count) == x86_64_mode_memory); |
| | 957 | } |
| | 958 | |
| 820 | #define REGN 6 | 959 | #define REGN 6 |
| 821 | static const uint8_t arg_regs[REGN] = { | 960 | static const uint8_t arg_regs[REGN] = { |
| 822 | TREG_RDI, TREG_RSI, TREG_RDX, TREG_RCX, TREG_R8, TREG_R9 | 961 | TREG_RDI, TREG_RSI, TREG_RDX, TREG_RCX, TREG_R8, TREG_R9 |
| @@ -827,7 +966,9 @@ static const uint8_t arg_regs[REGN] = { · 4 unmodified lines |
| 827 | parameters and the function address. */ | 966 | parameters and the function address. */ |
| 828 | void gfunc_call(int nb_args) | 967 | void gfunc_call(int nb_args) |
| 829 | { | 968 | { |
| 830 | int size, align, r, args_size, i; | 969 | X86_64_Mode mode; |
| | 970 | CType type; |
| | 971 | int size, align, r, args_size, i, j, reg_count; |
| 831 | int nb_reg_args = 0; | 972 | int nb_reg_args = 0; |
| 832 | int nb_sse_args = 0; | 973 | int nb_sse_args = 0; |
| 833 | int sse_reg, gen_reg; | 974 | int sse_reg, gen_reg; |
| @@ -835,17 +976,22 @@ void gfunc_call(int nb_args) · 1 unmodified line |
| 835 | /* calculate the number of integer/float arguments */ | 976 | /* calculate the number of integer/float arguments */ |
| 836 | args_size = 0; | 977 | args_size = 0; |
| 837 | for(i = 0; i < nb_args; i++) { | 978 | for(i = 0; i < nb_args; i++) { |
| 838 | if ((vtop[-i].type.t & VT_BTYPE) == VT_STRUCT) { | 979 | mode = classify_x86_64_arg(&vtop[-i].type, &size, ®_count); |
| 839 | args_size += type_size(&vtop[-i].type, &align); | 980 | switch (mode) { |
| 840 | args_size = (args_size + 7) & ~7; | 981 | case x86_64_mode_memory: |
| 841 | } else if ((vtop[-i].type.t & VT_BTYPE) == VT_LDOUBLE) { | 982 | case x86_64_mode_x87: |
| 842 | args_size += 16; | 983 | args_size += size; |
| 843 | } else if (is_sse_float(vtop[-i].type.t)) { | 984 | break; |
| 844 | nb_sse_args++; | 985 | |
| 845 | if (nb_sse_args > 8) args_size += 8; | 986 | case x86_64_mode_sse: |
| 846 | } else { | 987 | nb_sse_args += reg_count; |
| 847 | nb_reg_args++; | 988 | if (nb_sse_args > 8) args_size += size; |
| 848 | if (nb_reg_args > REGN) args_size += 8; | 989 | break; |
| | 990 | |
| | 991 | case x86_64_mode_integer: |
| | 992 | nb_reg_args += reg_count; |
| | 993 | if (nb_reg_args > REGN) args_size += size; |
| | 994 | break; |
| 849 | } | 995 | } |
| 850 | } | 996 | } |
| 851 | | 997 | |
| @@ -875,10 +1021,9 @@ void gfunc_call(int nb_args) · 23 unmodified lines |
| 875 | SValue tmp = vtop[0]; | 1021 | SValue tmp = vtop[0]; |
| 876 | vtop[0] = vtop[-i]; | 1022 | vtop[0] = vtop[-i]; |
| 877 | vtop[-i] = tmp; | 1023 | vtop[-i] = tmp; |
| 878 | if ((vtop->type.t & VT_BTYPE) == VT_STRUCT) { | 1024 | mode = classify_x86_64_arg(&vtop->type, &size, ®_count); |
| 879 | size = type_size(&vtop->type, &align); | 1025 | switch (mode) { |
| 880 | /* align to stack align size */ | 1026 | case x86_64_mode_memory: |
| 881 | size = (size + 7) & ~7; | | |
| 882 | /* allocate the necessary size on stack */ | 1027 | /* allocate the necessary size on stack */ |
| 883 | o(0x48); | 1028 | o(0x48); |
| 884 | oad(0xec81, size); /* sub $xxx, %rsp */ | 1029 | oad(0xec81, size); /* sub $xxx, %rsp */ |
| @@ -890,7 +1035,9 @@ void gfunc_call(int nb_args) · 5 unmodified lines |
| 890 | vswap(); | 1035 | vswap(); |
| 891 | vstore(); | 1036 | vstore(); |
| 892 | args_size += size; | 1037 | args_size += size; |
| 893 | } else if ((vtop->type.t & VT_BTYPE) == VT_LDOUBLE) { | 1038 | break; |
| | 1039 | |
| | 1040 | case x86_64_mode_x87: |
| 894 | gv(RC_ST0); | 1041 | gv(RC_ST0); |
| 895 | size = LDOUBLE_SIZE; | 1042 | size = LDOUBLE_SIZE; |
| 896 | oad(0xec8148, size); /* sub $xxx, %rsp */ | 1043 | oad(0xec8148, size); /* sub $xxx, %rsp */ |
| @@ -898,25 +1045,30 @@ void gfunc_call(int nb_args) · 1 unmodified line |
| 898 | g(0x24); | 1045 | g(0x24); |
| 899 | g(0x00); | 1046 | g(0x00); |
| 900 | args_size += size; | 1047 | args_size += size; |
| 901 | } else if (is_sse_float(vtop->type.t)) { | 1048 | break; |
| 902 | int j = --sse_reg; | 1049 | |
| 903 | if (j >= 8) { | 1050 | case x86_64_mode_sse: |
| | 1051 | if (sse_reg > 8) { |
| 904 | gv(RC_FLOAT); | 1052 | gv(RC_FLOAT); |
| 905 | o(0x50); /* push $rax */ | 1053 | o(0x50); /* push $rax */ |
| 906 | /* movq %xmm0, (%rsp) */ | 1054 | /* movq %xmm0, (%rsp) */ |
| 907 | o(0x04d60f66); | 1055 | o(0x04d60f66); |
| 908 | o(0x24); | 1056 | o(0x24); |
| 909 | args_size += 8; | 1057 | args_size += size; |
| 910 | } | 1058 | } |
| 911 | } else { | 1059 | sse_reg -= reg_count; |
| 912 | int j = --gen_reg; | 1060 | break; |
| | 1061 | |
| | 1062 | case x86_64_mode_integer: |
| 913 | /* simple type */ | 1063 | /* simple type */ |
| 914 | /* XXX: implicit cast ? */ | 1064 | /* XXX: implicit cast ? */ |
| 915 | if (j >= REGN) { | 1065 | if (gen_reg > REGN) { |
| 916 | r = gv(RC_INT); | 1066 | r = gv(RC_INT); |
| 917 | orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */ | 1067 | orex(0,r,0,0x50 + REG_VALUE(r)); /* push r */ |
| 918 | args_size += 8; | 1068 | args_size += size; |
| 919 | } | 1069 | } |
| | 1070 | gen_reg -= reg_count; |
| | 1071 | break; |
| 920 | } | 1072 | } |
| 921 | | 1073 | |
| 922 | /* And swap the argument back to it's original position. */ | 1074 | /* And swap the argument back to it's original position. */ |
| @@ -935,29 +1087,45 @@ void gfunc_call(int nb_args) · 12 unmodified lines |
| 935 | gen_reg = nb_reg_args; | 1087 | gen_reg = nb_reg_args; |
| 936 | sse_reg = nb_sse_args; | 1088 | sse_reg = nb_sse_args; |
| 937 | for(i = 0; i < nb_args; i++) { | 1089 | for(i = 0; i < nb_args; i++) { |
| 938 | if ((vtop->type.t & VT_BTYPE) == VT_STRUCT || | 1090 | mode = classify_x86_64_arg_type(&vtop->type, &type, &size, ®_count); |
| 939 | (vtop->type.t & VT_BTYPE) == VT_LDOUBLE) { | 1091 | /* Alter stack entry type so that gv() knows how to treat it */ |
| 940 | } else if (is_sse_float(vtop->type.t)) { | 1092 | vtop->type = type; |
| 941 | int j = --sse_reg; | 1093 | switch (mode) { |
| 942 | if (j < 8) { | 1094 | default: |
| 943 | gv(RC_FLOAT); /* only one float register */ | 1095 | break; |
| 944 | /* movaps %xmm0, %xmmN */ | 1096 | |
| 945 | o(0x280f); | 1097 | case x86_64_mode_sse: |
| 946 | o(0xc0 + (sse_reg << 3)); | 1098 | if (sse_reg > 8) { |
| | 1099 | sse_reg -= reg_count; |
| | 1100 | } else { |
| | 1101 | for (j = 0; j < reg_count; ++j) { |
| | 1102 | --sse_reg; |
| | 1103 | gv(RC_FLOAT); /* only one float register */ |
| | 1104 | /* movaps %xmm0, %xmmN */ |
| | 1105 | o(0x280f); |
| | 1106 | o(0xc0 + (sse_reg << 3)); |
| | 1107 | } |
| 947 | } | 1108 | } |
| 948 | } else { | 1109 | break; |
| 949 | int j = --gen_reg; | 1110 | |
| | 1111 | case x86_64_mode_integer: |
| 950 | /* simple type */ | 1112 | /* simple type */ |
| 951 | /* XXX: implicit cast ? */ | 1113 | /* XXX: implicit cast ? */ |
| 952 | if (j < REGN) { | 1114 | if (gen_reg > 8) { |
| 953 | int d = arg_regs[j]; | 1115 | gen_reg -= reg_count; |
| 954 | r = gv(RC_INT); | 1116 | } else { |
| 955 | if (j == 2 || j == 3) | 1117 | for (j = 0; j < reg_count; ++j) { |
| 956 | /* j=2: r10, j=3: r11 */ | 1118 | --gen_reg; |
| 957 | d = j + 8; | 1119 | int d = arg_regs[gen_reg]; |
| 958 | orex(1,d,r,0x89); /* mov */ | 1120 | r = gv(RC_INT); |
| 959 | o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d)); | 1121 | if (gen_reg == 2 || gen_reg == 3) |
| | 1122 | /* gen_reg=2: r10, gen_reg=3: r11 */ |
| | 1123 | d = gen_reg + 8; |
| | 1124 | orex(1,d,r,0x89); /* mov */ |
| | 1125 | o(0xc0 + REG_VALUE(r) * 8 + REG_VALUE(d)); |
| | 1126 | } |
| 960 | } | 1127 | } |
| | 1128 | break; |
| 961 | } | 1129 | } |
| 962 | vtop--; | 1130 | vtop--; |
| 963 | } | 1131 | } |
| @@ -994,7 +1162,8 @@ static void push_arg_reg(int i) { · 30 unmodified lines |
| 994 | /* generate function prolog of type 't' */ | 1162 | /* generate function prolog of type 't' */ |
| 995 | void gfunc_prolog(CType *func_type) | 1163 | void gfunc_prolog(CType *func_type) |
| 996 | { | 1164 | { |
| 997 | int i, addr, align, size; | 1165 | X86_64_Mode mode; |
| | 1166 | int i, addr, align, size, reg_count; |
| 998 | int param_index, param_addr, reg_param_index, sse_param_index; | 1167 | int param_index, param_addr, reg_param_index, sse_param_index; |
| 999 | Sym *sym; | 1168 | Sym *sym; |
| 1000 | CType *type; | 1169 | CType *type; |
| @@ -1070,7 +1239,8 @@ void gfunc_prolog(CType *func_type) · 69 unmodified lines |
| 1070 | /* if the function returns a structure, then add an | 1239 | /* if the function returns a structure, then add an |
| 1071 | implicit pointer parameter */ | 1240 | implicit pointer parameter */ |
| 1072 | func_vt = sym->type; | 1241 | func_vt = sym->type; |
| 1073 | if ((func_vt.t & VT_BTYPE) == VT_STRUCT) { | 1242 | mode = classify_x86_64_arg(&func_vt, &size, ®_count); |
| | 1243 | if (mode == x86_64_mode_memory) { |
| 1074 | push_arg_reg(reg_param_index); | 1244 | push_arg_reg(reg_param_index); |
| 1075 | param_addr = loc; | 1245 | param_addr = loc; |
| 1076 | | 1246 | |
| @@ -1081,35 +1251,46 @@ void gfunc_prolog(CType *func_type) · 4 unmodified lines |
| 1081 | /* define parameters */ | 1251 | /* define parameters */ |
| 1082 | while ((sym = sym->next) != NULL) { | 1252 | while ((sym = sym->next) != NULL) { |
| 1083 | type = &sym->type; | 1253 | type = &sym->type; |
| 1084 | size = type_size(type, &align); | 1254 | mode = classify_x86_64_arg(type, &size, ®_count); |
| 1085 | size = (size + 7) & ~7; | 1255 | switch (mode) { |
| 1086 | if (is_sse_float(type->t)) { | 1256 | case x86_64_mode_sse: |
| 1087 | if (sse_param_index < 8) { | 1257 | if (sse_param_index + reg_count <= 8) { |
| 1088 | /* save arguments passed by register */ | 1258 | /* save arguments passed by register */ |
| 1089 | loc -= 8; | 1259 | for (i = 0; i < reg_count; ++i) { |
| 1090 | o(0xd60f66); /* movq */ | 1260 | loc -= 8; |
| 1091 | gen_modrm(sse_param_index, VT_LOCAL, NULL, loc); | 1261 | o(0xd60f66); /* movq */ |
| | 1262 | gen_modrm(sse_param_index, VT_LOCAL, NULL, loc); |
| | 1263 | ++sse_param_index; |
| | 1264 | } |
| 1092 | param_addr = loc; | 1265 | param_addr = loc; |
| 1093 | } else { | 1266 | } else { |
| 1094 | param_addr = addr; | 1267 | param_addr = addr; |
| 1095 | addr += size; | 1268 | addr += size; |
| | 1269 | sse_param_index += reg_count; |
| 1096 | } | 1270 | } |
| 1097 | sse_param_index++; | 1271 | break; |
| 1098 | | 1272 | |
| 1099 | } else if ((type->t & VT_BTYPE) == VT_STRUCT || | 1273 | case x86_64_mode_memory: |
| 1100 | (type->t & VT_BTYPE) == VT_LDOUBLE) { | 1274 | case x86_64_mode_x87: |
| 1101 | param_addr = addr; | 1275 | param_addr = addr; |
| 1102 | addr += size; | 1276 | addr += size; |
| 1103 | } else { | 1277 | break; |
| 1104 | if (reg_param_index < REGN) { | 1278 | |
| | 1279 | case x86_64_mode_integer: { |
| | 1280 | if (reg_param_index + reg_count <= REGN) { |
| 1105 | /* save arguments passed by register */ | 1281 | /* save arguments passed by register */ |
| 1106 | push_arg_reg(reg_param_index); | 1282 | for (i = 0; i < reg_count; ++i) { |
| | 1283 | push_arg_reg(reg_param_index); |
| | 1284 | ++reg_param_index; |
| | 1285 | } |
| 1107 | param_addr = loc; | 1286 | param_addr = loc; |
| 1108 | } else { | 1287 | } else { |
| 1109 | param_addr = addr; | 1288 | param_addr = addr; |
| 1110 | addr += 8; | 1289 | addr += size; |
| | 1290 | reg_param_index += reg_count; |
| 1111 | } | 1291 | } |
| 1112 | reg_param_index++; | 1292 | break; |
| | 1293 | } |
| 1113 | } | 1294 | } |
| 1114 | sym_push(sym->v & ~SYM_FIELD, type, | 1295 | sym_push(sym->v & ~SYM_FIELD, type, |
| 1115 | VT_LOCAL | VT_LVAL, param_addr); | 1296 | VT_LOCAL | VT_LVAL, param_addr); |