Unified · Split
| @@ -80,6 +80,9 @@ · 79 unmodified lines | |||
| 80 | 80 | ||
| 81 | #define TOK_HASH_SIZE 2048 /* must be a power of two */ | 81 | #define TOK_HASH_SIZE 2048 /* must be a power of two */ |
| 82 | #define TOK_ALLOC_INCR 512 /* must be a power of two */ | 82 | #define TOK_ALLOC_INCR 512 /* must be a power of two */ |
| 83 | #define TOK_STR_ALLOC_INCR_BITS 6 | ||
| 84 | #define TOK_STR_ALLOC_INCR (1 << TOK_STR_ALLOC_INCR_BITS) | ||
| 85 | #define TOK_MAX_SIZE 4 /* token max size in int unit when stored in string */ | ||
| 83 | 86 | ||
| 84 | /* token symbol management */ | 87 | /* token symbol management */ |
| 85 | typedef struct TokenSym { | 88 | typedef struct TokenSym { |
| @@ -238,6 +241,7 @@ typedef struct ParseState { · 152 unmodified lines | |||
| 238 | typedef struct TokenString { | 241 | typedef struct TokenString { |
| 239 | int *str; | 242 | int *str; |
| 240 | int len; | 243 | int len; |
| 244 | int allocated_len; | ||
| 241 | int last_line_num; | 245 | int last_line_num; |
| 242 | } TokenString; | 246 | } TokenString; |
| 243 | 247 | ||
| @@ -1115,26 +1119,11 @@ void test_lvalue(void) · 871 unmodified lines | |||
| 1115 | expect("lvalue"); | 1119 | expect("lvalue"); |
| 1116 | } | 1120 | } |
| 1117 | 1121 | ||
| 1118 | TokenSym *tok_alloc(const char *str, int len) | 1122 | /* allocate a new token */ |
| 1123 | static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len) | ||
| 1119 | { | 1124 | { |
| 1120 | TokenSym *ts, **pts, **ptable; | 1125 | TokenSym *ts, **ptable; |
| 1121 | int i; | 1126 | int i; |
| 1122 | unsigned int h; | ||
| 1123 | |||
| 1124 | h = 1; | ||
| 1125 | for(i=0;i<len;i++) | ||
| 1126 | h = h * 263 + ((unsigned char *)str)[i]; | ||
| 1127 | h &= (TOK_HASH_SIZE - 1); | ||
| 1128 | |||
| 1129 | pts = &hash_ident[h]; | ||
| 1130 | while (1) { | ||
| 1131 | ts = *pts; | ||
| 1132 | if (!ts) | ||
| 1133 | break; | ||
| 1134 | if (ts->len == len && !memcmp(ts->str, str, len)) | ||
| 1135 | return ts; | ||
| 1136 | pts = &(ts->hash_next); | ||
| 1137 | } | ||
| 1138 | 1127 | ||
| 1139 | if (tok_ident >= SYM_FIRST_ANOM) | 1128 | if (tok_ident >= SYM_FIRST_ANOM) |
| 1140 | error("memory full"); | 1129 | error("memory full"); |
| @@ -1163,6 +1152,33 @@ TokenSym *tok_alloc(const char *str, int len) · 22 unmodified lines | |||
| 1163 | return ts; | 1152 | return ts; |
| 1164 | } | 1153 | } |
| 1165 | 1154 | ||
| 1155 | #define TOK_HASH_INIT 1 | ||
| 1156 | #define TOK_HASH_FUNC(h, c) ((h) * 263 + (c)) | ||
| 1157 | |||
| 1158 | /* find a token and add it if not found */ | ||
| 1159 | static TokenSym *tok_alloc(const char *str, int len) | ||
| 1160 | { | ||
| 1161 | TokenSym *ts, **pts; | ||
| 1162 | int i; | ||
| 1163 | unsigned int h; | ||
| 1164 | |||
| 1165 | h = TOK_HASH_INIT; | ||
| 1166 | for(i=0;i<len;i++) | ||
| 1167 | h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]); | ||
| 1168 | h &= (TOK_HASH_SIZE - 1); | ||
| 1169 | |||
| 1170 | pts = &hash_ident[h]; | ||
| 1171 | for(;;) { | ||
| 1172 | ts = *pts; | ||
| 1173 | if (!ts) | ||
| 1174 | break; | ||
| 1175 | if (ts->len == len && !memcmp(ts->str, str, len)) | ||
| 1176 | return ts; | ||
| 1177 | pts = &(ts->hash_next); | ||
| 1178 | } | ||
| 1179 | return tok_alloc_new(pts, str, len); | ||
| 1180 | } | ||
| 1181 | |||
| 1166 | /* CString handling */ | 1182 | /* CString handling */ |
| 1167 | 1183 | ||
| 1168 | static void cstr_realloc(CString *cstr, int new_size) | 1184 | static void cstr_realloc(CString *cstr, int new_size) |
| @@ -1904,6 +1920,7 @@ static inline void tok_str_new(TokenString *s) · 735 unmodified lines | |||
| 1904 | { | 1920 | { |
| 1905 | s->str = NULL; | 1921 | s->str = NULL; |
| 1906 | s->len = 0; | 1922 | s->len = 0; |
| 1923 | s->allocated_len = 0; | ||
| 1907 | s->last_line_num = -1; | 1924 | s->last_line_num = -1; |
| 1908 | } | 1925 | } |
| 1909 | 1926 | ||
| @@ -1915,52 +1932,114 @@ static void tok_str_free(int *str) · 5 unmodified lines | |||
| 1915 | 1932 | ||
| 1916 | p = str; | 1933 | p = str; |
| 1917 | for(;;) { | 1934 | for(;;) { |
| 1918 | t = *p++; | 1935 | t = *p; |
| 1936 | /* NOTE: we test zero separately so that GCC can generate a | ||
| 1937 | table for the following switch */ | ||
| 1919 | if (t == 0) | 1938 | if (t == 0) |
| 1920 | break; | 1939 | break; |
| 1921 | if (t == TOK_STR || t == TOK_LSTR || t == TOK_PPNUM) { | 1940 | switch(t) { |
| 1941 | case TOK_CINT: | ||
| 1942 | case TOK_CUINT: | ||
| 1943 | case TOK_CCHAR: | ||
| 1944 | case TOK_LCHAR: | ||
| 1945 | case TOK_CFLOAT: | ||
| 1946 | case TOK_LINENUM: | ||
| 1947 | p += 2; | ||
| 1948 | break; | ||
| 1949 | case TOK_PPNUM: | ||
| 1950 | case TOK_STR: | ||
| 1951 | case TOK_LSTR: | ||
| 1922 | /* XXX: use a macro to be portable on 64 bit ? */ | 1952 | /* XXX: use a macro to be portable on 64 bit ? */ |
| 1923 | cstr = (CString *)(*p++); | 1953 | cstr = (CString *)p[1]; |
| 1924 | cstr_free(cstr); | 1954 | cstr_free(cstr); |
| 1925 | tcc_free(cstr); | 1955 | tcc_free(cstr); |
| 1926 | } else { | 1956 | p += 2; |
| 1927 | p += tok_ext_size(t); | 1957 | break; |
| 1958 | case TOK_CDOUBLE: | ||
| 1959 | case TOK_CLLONG: | ||
| 1960 | case TOK_CULLONG: | ||
| 1961 | p += 3; | ||
| 1962 | break; | ||
| 1963 | case TOK_CLDOUBLE: | ||
| 1964 | p += 1 + (LDOUBLE_SIZE / 4); | ||
| 1965 | break; | ||
| 1966 | default: | ||
| 1967 | p++; | ||
| 1968 | break; | ||
| 1928 | } | 1969 | } |
| 1929 | } | 1970 | } |
| 1930 | tcc_free(str); | 1971 | tcc_free(str); |
| 1931 | } | 1972 | } |
| 1932 | 1973 | ||
| 1974 | static int *tok_str_realloc(TokenString *s) | ||
| 1975 | { | ||
| 1976 | int *str, len; | ||
| 1977 | |||
| 1978 | len = s->allocated_len + TOK_STR_ALLOC_INCR; | ||
| 1979 | str = tcc_realloc(s->str, len * sizeof(int)); | ||
| 1980 | if (!str) | ||
| 1981 | error("memory full"); | ||
| 1982 | s->allocated_len = len; | ||
| 1983 | s->str = str; | ||
| 1984 | return str; | ||
| 1985 | } | ||
| 1986 | |||
| 1933 | static void tok_str_add(TokenString *s, int t) | 1987 | static void tok_str_add(TokenString *s, int t) |
| 1934 | { | 1988 | { |
| 1935 | int len, *str; | 1989 | int len, *str; |
| 1936 | 1990 | ||
| 1937 | len = s->len; | 1991 | len = s->len; |
| 1938 | str = s->str; | 1992 | str = s->str; |
| 1939 | if ((len & 63) == 0) { | 1993 | if (len >= s->allocated_len) |
| 1940 | str = tcc_realloc(str, (len + 64) * sizeof(int)); | 1994 | str = tok_str_realloc(s); |
| 1941 | if (!str) | ||
| 1942 | return; | ||
| 1943 | s->str = str; | ||
| 1944 | } | ||
| 1945 | str[len++] = t; | 1995 | str[len++] = t; |
| 1946 | s->len = len; | 1996 | s->len = len; |
| 1947 | } | 1997 | } |
| 1948 | 1998 | ||
| 1949 | static void tok_str_add2(TokenString *s, int t, CValue *cv) | 1999 | static void tok_str_add2(TokenString *s, int t, CValue *cv) |
| 1950 | { | 2000 | { |
| 1951 | int n, i; | 2001 | int len, *str; |
| 1952 | CValue cv1; | ||
| 1953 | 2002 | ||
| 1954 | tok_str_add(s, t); | 2003 | len = s->len; |
| 1955 | if (t == TOK_STR || t == TOK_LSTR || t == TOK_PPNUM) { | 2004 | str = s->str; |
| 1956 | /* special case: need to duplicate string */ | 2005 | |
| 1957 | cv1.cstr = cstr_dup(cv->cstr); | 2006 | /* allocate space for worst case */ |
| 1958 | tok_str_add(s, cv1.tab[0]); | 2007 | if (len + TOK_MAX_SIZE > s->allocated_len) |
| 1959 | } else { | 2008 | str = tok_str_realloc(s); |
| 1960 | n = tok_ext_size(t); | 2009 | str[len++] = t; |
| 1961 | for(i=0;i<n;i++) | 2010 | switch(t) { |
| 1962 | tok_str_add(s, cv->tab[i]); | 2011 | case TOK_CINT: |
| 2012 | case TOK_CUINT: | ||
| 2013 | case TOK_CCHAR: | ||
| 2014 | case TOK_LCHAR: | ||
| 2015 | case TOK_CFLOAT: | ||
| 2016 | case TOK_LINENUM: | ||
| 2017 | str[len++] = cv->tab[0]; | ||
| 2018 | break; | ||
| 2019 | case TOK_PPNUM: | ||
| 2020 | case TOK_STR: | ||
| 2021 | case TOK_LSTR: | ||
| 2022 | str[len++] = (int)cstr_dup(cv->cstr); | ||
| 2023 | break; | ||
| 2024 | case TOK_CDOUBLE: | ||
| 2025 | case TOK_CLLONG: | ||
| 2026 | case TOK_CULLONG: | ||
| 2027 | str[len++] = cv->tab[0]; | ||
| 2028 | str[len++] = cv->tab[1]; | ||
| 2029 | break; | ||
| 2030 | case TOK_CLDOUBLE: | ||
| 2031 | #if LDOUBLE_SIZE == 12 | ||
| 2032 | str[len++] = cv->tab[0]; | ||
| 2033 | str[len++] = cv->tab[1]; | ||
| 2034 | str[len++] = cv->tab[2]; | ||
| 2035 | #else | ||
| 2036 | #error add long double size support | ||
| 2037 | #endif | ||
| 2038 | break; | ||
| 2039 | default: | ||
| 2040 | break; | ||
| 1963 | } | 2041 | } |
| 2042 | s->len = len; | ||
| 1964 | } | 2043 | } |
| 1965 | 2044 | ||
| 1966 | /* add the current parse token in token string 's' */ | 2045 | /* add the current parse token in token string 's' */ |
| @@ -1977,18 +2056,47 @@ static void tok_str_add_tok(TokenString *s) · 10 unmodified lines | |||
| 1977 | tok_str_add2(s, tok, &tokc); | 2056 | tok_str_add2(s, tok, &tokc); |
| 1978 | } | 2057 | } |
| 1979 | 2058 | ||
| 1980 | /* get a token from an integer array and increment pointer accordingly */ | 2059 | #if LDOUBLE_SIZE == 12 |
| 1981 | static int tok_get(int **tok_str, CValue *cv) | 2060 | #define LDOUBLE_GET(p, cv) \ |
| 1982 | { | 2061 | cv.tab[0] = p[0]; \ |
| 1983 | int *p, t, n, i; | 2062 | cv.tab[1] = p[1]; \ |
| 2063 | cv.tab[2] = p[2]; | ||
| 2064 | #else | ||
| 2065 | #error add long double size support | ||
| 2066 | #endif | ||
| 2067 | |||
| 1984 | 2068 | ||
| 1985 | p = *tok_str; | 2069 | /* get a token from an integer array and increment pointer |
| 1986 | t = *p++; | 2070 | accordingly. we code it as a macro to avoid pointer aliasing. */ |
| 1987 | n = tok_ext_size(t); | 2071 | #define TOK_GET(t, p, cv) \ |
| 1988 | for(i=0;i<n;i++) | 2072 | { \ |
| 1989 | cv->tab[i] = *p++; | 2073 | t = *p++; \ |
| 1990 | *tok_str = p; | 2074 | switch(t) { \ |
| 1991 | return t; | 2075 | case TOK_CINT: \ |
| 2076 | case TOK_CUINT: \ | ||
| 2077 | case TOK_CCHAR: \ | ||
| 2078 | case TOK_LCHAR: \ | ||
| 2079 | case TOK_CFLOAT: \ | ||
| 2080 | case TOK_LINENUM: \ | ||
| 2081 | case TOK_STR: \ | ||
| 2082 | case TOK_LSTR: \ | ||
| 2083 | case TOK_PPNUM: \ | ||
| 2084 | cv.tab[0] = *p++; \ | ||
| 2085 | break; \ | ||
| 2086 | case TOK_CDOUBLE: \ | ||
| 2087 | case TOK_CLLONG: \ | ||
| 2088 | case TOK_CULLONG: \ | ||
| 2089 | cv.tab[0] = p[0]; \ | ||
| 2090 | cv.tab[1] = p[1]; \ | ||
| 2091 | p += 2; \ | ||
| 2092 | break; \ | ||
| 2093 | case TOK_CLDOUBLE: \ | ||
| 2094 | LDOUBLE_GET(p, cv); \ | ||
| 2095 | p += LDOUBLE_SIZE / 4; \ | ||
| 2096 | break; \ | ||
| 2097 | default: \ | ||
| 2098 | break; \ | ||
| 2099 | } \ | ||
| 1992 | } | 2100 | } |
| 1993 | 2101 | ||
| 1994 | /* defines handling */ | 2102 | /* defines handling */ |
| @@ -2059,7 +2167,7 @@ static Sym *label_push(int v, int flags) · 64 unmodified lines | |||
| 2059 | } | 2167 | } |
| 2060 | 2168 | ||
| 2061 | /* eval an expression for #if/#elif */ | 2169 | /* eval an expression for #if/#elif */ |
| 2062 | int expr_preprocess(void) | 2170 | static int expr_preprocess(void) |
| 2063 | { | 2171 | { |
| 2064 | int c, t; | 2172 | int c, t; |
| 2065 | TokenString str; | 2173 | TokenString str; |
| @@ -2096,13 +2204,13 @@ int expr_preprocess(void) · 30 unmodified lines | |||
| 2096 | } | 2204 | } |
| 2097 | 2205 | ||
| 2098 | #if defined(PARSE_DEBUG) || defined(PP_DEBUG) | 2206 | #if defined(PARSE_DEBUG) || defined(PP_DEBUG) |
| 2099 | void tok_print(int *str) | 2207 | static void tok_print(int *str) |
| 2100 | { | 2208 | { |
| 2101 | int t; | 2209 | int t; |
| 2102 | CValue cval; | 2210 | CValue cval; |
| 2103 | 2211 | ||
| 2104 | while (1) { | 2212 | while (1) { |
| 2105 | t = tok_get(&str, &cval); | 2213 | TOK_GET(t, str, cval); |
| 2106 | if (!t) | 2214 | if (!t) |
| 2107 | break; | 2215 | break; |
| 2108 | printf(" %s", get_tok_str(t, &cval)); | 2216 | printf(" %s", get_tok_str(t, &cval)); |
| @@ -2876,6 +2984,7 @@ static inline void next_nomacro1(void) · 767 unmodified lines | |||
| 2876 | int b, t, c; | 2984 | int b, t, c; |
| 2877 | TokenSym *ts; | 2985 | TokenSym *ts; |
| 2878 | uint8_t *p, *p1; | 2986 | uint8_t *p, *p1; |
| 2987 | unsigned int h; | ||
| 2879 | 2988 | ||
| 2880 | p = file->buf_ptr; | 2989 | p = file->buf_ptr; |
| 2881 | redo_no_start: | 2990 | redo_no_start: |
| @@ -2944,10 +3053,10 @@ static inline void next_nomacro1(void) · 62 unmodified lines | |||
| 2944 | break; | 3053 | break; |
| 2945 | 3054 | ||
| 2946 | case '\n': | 3055 | case '\n': |
| 2947 | file->line_num++; | ||
| 2948 | if (return_linefeed) { | 3056 | if (return_linefeed) { |
| 2949 | tok = TOK_LINEFEED; | 3057 | tok = TOK_LINEFEED; |
| 2950 | } else { | 3058 | } else { |
| 3059 | file->line_num++; | ||
| 2951 | tok_flags |= TOK_FLAG_BOL; | 3060 | tok_flags |= TOK_FLAG_BOL; |
| 2952 | p++; | 3061 | p++; |
| 2953 | goto redo_no_start; | 3062 | goto redo_no_start; |
| @@ -2989,16 +3098,35 @@ static inline void next_nomacro1(void) · 35 unmodified lines | |||
| 2989 | case '_': | 3098 | case '_': |
| 2990 | parse_ident_fast: | 3099 | parse_ident_fast: |
| 2991 | p1 = p; | 3100 | p1 = p; |
| 3101 | h = TOK_HASH_INIT; | ||
| 3102 | h = TOK_HASH_FUNC(h, c); | ||
| 2992 | p++; | 3103 | p++; |
| 2993 | for(;;) { | 3104 | for(;;) { |
| 2994 | c = *p; | 3105 | c = *p; |
| 2995 | if (!isid(c) && !isnum(c)) | 3106 | if (!isid(c) && !isnum(c)) |
| 2996 | break; | 3107 | break; |
| 3108 | h = TOK_HASH_FUNC(h, c); | ||
| 2997 | p++; | 3109 | p++; |
| 2998 | } | 3110 | } |
| 2999 | if (c != '\\') { | 3111 | if (c != '\\') { |
| 3000 | /* fast case : no stray found, so we have the full token */ | 3112 | TokenSym **pts; |
| 3001 | ts = tok_alloc(p1, p - p1); | 3113 | int len; |
| 3114 | |||
| 3115 | /* fast case : no stray found, so we have the full token | ||
| 3116 | and we have already hashed it */ | ||
| 3117 | len = p - p1; | ||
| 3118 | h &= (TOK_HASH_SIZE - 1); | ||
| 3119 | pts = &hash_ident[h]; | ||
| 3120 | for(;;) { | ||
| 3121 | ts = *pts; | ||
| 3122 | if (!ts) | ||
| 3123 | break; | ||
| 3124 | if (ts->len == len && !memcmp(ts->str, p1, len)) | ||
| 3125 | goto token_found; | ||
| 3126 | pts = &(ts->hash_next); | ||
| 3127 | } | ||
| 3128 | ts = tok_alloc_new(pts, p1, len); | ||
| 3129 | token_found: ; | ||
| 3002 | } else { | 3130 | } else { |
| 3003 | /* slower case */ | 3131 | /* slower case */ |
| 3004 | cstr_reset(&tokcstr); | 3132 | cstr_reset(&tokcstr); |
| @@ -3019,8 +3147,8 @@ static inline void next_nomacro1(void) · 14 unmodified lines | |||
| 3019 | tok = ts->tok; | 3147 | tok = ts->tok; |
| 3020 | break; | 3148 | break; |
| 3021 | case 'L': | 3149 | case 'L': |
| 3022 | c = p[1]; | 3150 | t = p[1]; |
| 3023 | if (c != '\\' && c != '\'' && c != '\"') { | 3151 | if (t != '\\' && t != '\'' && t != '\"') { |
| 3024 | /* fast case */ | 3152 | /* fast case */ |
| 3025 | goto parse_ident_fast; | 3153 | goto parse_ident_fast; |
| 3026 | } else { | 3154 | } else { |
| @@ -3268,7 +3396,7 @@ static void next_nomacro(void) · 241 unmodified lines | |||
| 3268 | redo: | 3396 | redo: |
| 3269 | tok = *macro_ptr; | 3397 | tok = *macro_ptr; |
| 3270 | if (tok) { | 3398 | if (tok) { |
| 3271 | tok = tok_get(¯o_ptr, &tokc); | 3399 | TOK_GET(tok, macro_ptr, tokc); |
| 3272 | if (tok == TOK_LINENUM) { | 3400 | if (tok == TOK_LINENUM) { |
| 3273 | file->line_num = tokc.i; | 3401 | file->line_num = tokc.i; |
| 3274 | goto redo; | 3402 | goto redo; |
| @@ -3291,12 +3419,12 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 16 unmodified lines | |||
| 3291 | tok_str_new(&str); | 3419 | tok_str_new(&str); |
| 3292 | last_tok = 0; | 3420 | last_tok = 0; |
| 3293 | while(1) { | 3421 | while(1) { |
| 3294 | t = tok_get(¯o_str, &cval); | 3422 | TOK_GET(t, macro_str, cval); |
| 3295 | if (!t) | 3423 | if (!t) |
| 3296 | break; | 3424 | break; |
| 3297 | if (t == '#') { | 3425 | if (t == '#') { |
| 3298 | /* stringize */ | 3426 | /* stringize */ |
| 3299 | t = tok_get(¯o_str, &cval); | 3427 | TOK_GET(t, macro_str, cval); |
| 3300 | if (!t) | 3428 | if (!t) |
| 3301 | break; | 3429 | break; |
| 3302 | s = sym_find2(args, t); | 3430 | s = sym_find2(args, t); |
| @@ -3307,7 +3435,7 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 4 unmodified lines | |||
| 3307 | while (*st) { | 3435 | while (*st) { |
| 3308 | if (notfirst) | 3436 | if (notfirst) |
| 3309 | cstr_ccat(&cstr, ' '); | 3437 | cstr_ccat(&cstr, ' '); |
| 3310 | t = tok_get(&st, &cval); | 3438 | TOK_GET(t, st, cval); |
| 3311 | cstr_cat(&cstr, get_tok_str(t, &cval)); | 3439 | cstr_cat(&cstr, get_tok_str(t, &cval)); |
| 3312 | notfirst = 1; | 3440 | notfirst = 1; |
| 3313 | } | 3441 | } |
| @@ -3348,7 +3476,7 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 34 unmodified lines | |||
| 3348 | int t1; | 3476 | int t1; |
| 3349 | add_var: | 3477 | add_var: |
| 3350 | for(;;) { | 3478 | for(;;) { |
| 3351 | t1 = tok_get(&st, &cval); | 3479 | TOK_GET(t1, st, cval); |
| 3352 | if (!t1) | 3480 | if (!t1) |
| 3353 | break; | 3481 | break; |
| 3354 | tok_str_add2(&str, t1, &cval); | 3482 | tok_str_add2(&str, t1, &cval); |
| @@ -3392,7 +3520,7 @@ static int *macro_twosharps(void) · 37 unmodified lines | |||
| 3392 | macro_ptr1 = macro_ptr; | 3520 | macro_ptr1 = macro_ptr; |
| 3393 | t = *macro_ptr; | 3521 | t = *macro_ptr; |
| 3394 | if (t) { | 3522 | if (t) { |
| 3395 | t = tok_get(¯o_ptr, &cval); | 3523 | TOK_GET(t, macro_ptr, cval); |
| 3396 | 3524 | ||
| 3397 | /* We concatenate the two tokens if we have an | 3525 | /* We concatenate the two tokens if we have an |
| 3398 | identifier or a preprocessing number */ | 3526 | identifier or a preprocessing number */ |