This commit fixed the problem that TCC directly cast each byte in wide string literal to wchar_t, which is wrong when wide string literal contains real wide chars. It fixed the problem by assuming input charset is UTF-8, and wchar_t stores wide chars in UTF-16 (Windows) or UTF-32 (others). The UTF-8 decoder is coded according to The Unicode Standard Version 10.
Unified · Split
| @@ -2105,13 +2105,79 @@ static void parse_escape_string(CString *outstr, const uint8_t *buf, int is_long · 2104 unmodified lines | |||
| 2105 | tcc_warning("unknown escape sequence: \'\\x%x\'", c); | 2105 | tcc_warning("unknown escape sequence: \'\\x%x\'", c); |
| 2106 | break; | 2106 | break; |
| 2107 | } | 2107 | } |
| 2108 | } else if (is_long && c >= 0x80) { | ||
| 2109 | /* assume we are processing UTF-8 sequence */ | ||
| 2110 | /* reference: The Unicode Standard, Version 10.0, ch3.9 */ | ||
| 2111 | |||
| 2112 | int cont; /* count of continuation bytes */ | ||
| 2113 | int skip; /* how many bytes should skip when error occured */ | ||
| 2114 | int i; | ||
| 2115 | |||
| 2116 | /* decode leading byte */ | ||
| 2117 | if (c < 0xC2) { | ||
| 2118 | skip = 1; goto invalid_utf8_sequence; | ||
| 2119 | } else if (c <= 0xDF) { | ||
| 2120 | cont = 1; n = c & 0x1f; | ||
| 2121 | } else if (c <= 0xEF) { | ||
| 2122 | cont = 2; n = c & 0xf; | ||
| 2123 | } else if (c <= 0xF4) { | ||
| 2124 | cont = 3; n = c & 0x7; | ||
| 2125 | } else { | ||
| 2126 | skip = 1; goto invalid_utf8_sequence; | ||
| 2127 | } | ||
| 2128 | |||
| 2129 | /* decode continuation bytes */ | ||
| 2130 | for (i = 1; i <= cont; i++) { | ||
| 2131 | int l = 0x80, h = 0xBF; | ||
| 2132 | |||
| 2133 | /* adjust limit for second byte */ | ||
| 2134 | if (i == 1) { | ||
| 2135 | switch (c) { | ||
| 2136 | case 0xE0: l = 0xA0; break; | ||
| 2137 | case 0xED: h = 0x9F; break; | ||
| 2138 | case 0xF0: l = 0x90; break; | ||
| 2139 | case 0xF4: h = 0x8F; break; | ||
| 2140 | } | ||
| 2141 | } | ||
| 2142 | |||
| 2143 | if (p[i] < l || p[i] > h) { | ||
| 2144 | skip = i; goto invalid_utf8_sequence; | ||
| 2145 | } | ||
| 2146 | |||
| 2147 | n = (n << 6) | (p[i] & 0x3f); | ||
| 2148 | } | ||
| 2149 | |||
| 2150 | /* advance pointer */ | ||
| 2151 | p += 1 + cont; | ||
| 2152 | c = n; | ||
| 2153 | goto add_char_nonext; | ||
| 2154 | |||
| 2155 | /* error handling */ | ||
| 2156 | invalid_utf8_sequence: | ||
| 2157 | tcc_warning("ill-formed UTF-8 subsequence starting with: \'\\x%x\'", c); | ||
| 2158 | c = 0xFFFD; | ||
| 2159 | p += skip; | ||
| 2160 | goto add_char_nonext; | ||
| 2161 | |||
| 2108 | } | 2162 | } |
| 2109 | p++; | 2163 | p++; |
| 2110 | add_char_nonext: | 2164 | add_char_nonext: |
| 2111 | if (!is_long) | 2165 | if (!is_long) |
| 2112 | cstr_ccat(outstr, c); | 2166 | cstr_ccat(outstr, c); |
| 2113 | else | 2167 | else { |
| 2168 | #ifdef TCC_TARGET_PE | ||
| 2169 | /* store as UTF-16 */ | ||
| 2170 | if (c < 0x10000) { | ||
| 2171 | cstr_wccat(outstr, c); | ||
| 2172 | } else { | ||
| 2173 | c -= 0x10000; | ||
| 2174 | cstr_wccat(outstr, (c >> 10) + 0xD800); | ||
| 2175 | cstr_wccat(outstr, (c & 0x3FF) + 0xDC00); | ||
| 2176 | } | ||
| 2177 | #else | ||
| 2114 | cstr_wccat(outstr, c); | 2178 | cstr_wccat(outstr, c); |
| 2179 | #endif | ||
| 2180 | } | ||
| 2115 | } | 2181 | } |
| 2116 | /* add a trailing '\0' */ | 2182 | /* add a trailing '\0' */ |
| 2117 | if (!is_long) | 2183 | if (!is_long) |