This commit fixed the problem that TCC directly cast each byte in wide string literal to wchar_t, which is wrong when wide string literal contains real wide chars. It fixed the problem by assuming input charset is UTF-8, and wchar_t stores wide chars in UTF-16 (Windows) or UTF-32 (others). The UTF-8 decoder is coded according to The Unicode Standard Version 10.
Unified · Split
| @@ -2105,13 +2105,79 @@ static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long · 2104 unmodified lines | ||
| 2105 | 2105 | tcc_warning("unknown escape sequence: \'\\x%x\'", c); |
| 2106 | 2106 | break; |
| 2107 | 2107 | } |
| 2108 | + } else if (is_long && c >= 0x80) { | |
| 2109 | + /* assume we are processing UTF-8 sequence */ | |
| 2110 | + /* reference: The Unicode Standard, Version 10.0, ch3.9 */ | |
| 2111 | + | |
| 2112 | + int cont; /* count of continuation bytes */ | |
| 2113 | + int skip; /* how many bytes should skip when error occured */ | |
| 2114 | + int i; | |
| 2115 | + | |
| 2116 | + /* decode leading byte */ | |
| 2117 | + if (c < 0xC2) { | |
| 2118 | + skip = 1; goto invalid_utf8_sequence; | |
| 2119 | + } else if (c <= 0xDF) { | |
| 2120 | + cont = 1; n = c & 0x1f; | |
| 2121 | + } else if (c <= 0xEF) { | |
| 2122 | + cont = 2; n = c & 0xf; | |
| 2123 | + } else if (c <= 0xF4) { | |
| 2124 | + cont = 3; n = c & 0x7; | |
| 2125 | + } else { | |
| 2126 | + skip = 1; goto invalid_utf8_sequence; | |
| 2127 | + } | |
| 2128 | + | |
| 2129 | + /* decode continuation bytes */ | |
| 2130 | + for (i = 1; i <= cont; i++) { | |
| 2131 | + int l = 0x80, h = 0xBF; | |
| 2132 | + | |
| 2133 | + /* adjust limit for second byte */ | |
| 2134 | + if (i == 1) { | |
| 2135 | + switch (c) { | |
| 2136 | + case 0xE0: l = 0xA0; break; | |
| 2137 | + case 0xED: h = 0x9F; break; | |
| 2138 | + case 0xF0: l = 0x90; break; | |
| 2139 | + case 0xF4: h = 0x8F; break; | |
| 2140 | + } | |
| 2141 | + } | |
| 2142 | + | |
| 2143 | + if (p[i] < l || p[i] > h) { | |
| 2144 | + skip = i; goto invalid_utf8_sequence; | |
| 2145 | + } | |
| 2146 | + | |
| 2147 | + n = (n << 6) | (p[i] & 0x3f); | |
| 2148 | + } | |
| 2149 | + | |
| 2150 | + /* advance pointer */ | |
| 2151 | + p += 1 + cont; | |
| 2152 | + c = n; | |
| 2153 | + goto add_char_nonext; | |
| 2154 | + | |
| 2155 | + /* error handling */ | |
| 2156 | + invalid_utf8_sequence: | |
| 2157 | + tcc_warning("ill-formed UTF-8 subsequence starting with: \'\\x%x\'", c); | |
| 2158 | + c = 0xFFFD; | |
| 2159 | + p += skip; | |
| 2160 | + goto add_char_nonext; | |
| 2161 | + | |
| 2108 | 2162 | } |
| 2109 | 2163 | p++; |
| 2110 | 2164 | add_char_nonext: |
| 2111 | 2165 | if (!is_long) |
| 2112 | 2166 | cstr_ccat(outstr, c); |
| 2113 | − else | |
| 2167 | + else { | |
| 2168 | +#ifdef TCC_TARGET_PE | |
| 2169 | + /* store as UTF-16 */ | |
| 2170 | + if (c < 0x10000) { | |
| 2171 | + cstr_wccat(outstr, c); | |
| 2172 | + } else { | |
| 2173 | + c -= 0x10000; | |
| 2174 | + cstr_wccat(outstr, (c >> 10) + 0xD800); | |
| 2175 | + cstr_wccat(outstr, (c & 0x3FF) + 0xDC00); | |
| 2176 | + } | |
| 2177 | +#else | |
| 2114 | 2178 | cstr_wccat(outstr, c); |
| 2179 | +#endif | |
| 2180 | + } | |
| 2115 | 2181 | } |
| 2116 | 2182 | /* add a trailing '\0' */ |
| 2117 | 2183 | if (!is_long) |