History

optimized token strings - optimized token hashing

bellard committed 23 years ago

Commit 9e180471 parent 8901fbe1 file · +193 −65Browse files

Unified · Split

tcc.c+193−65
@@ -80,6 +80,9 @@ · 79 unmodified lines
80 80
81#define TOK_HASH_SIZE 2048 /* must be a power of two */81#define TOK_HASH_SIZE 2048 /* must be a power of two */
82#define TOK_ALLOC_INCR 512 /* must be a power of two */82#define TOK_ALLOC_INCR 512 /* must be a power of two */
83#define TOK_STR_ALLOC_INCR_BITS 6
84#define TOK_STR_ALLOC_INCR (1 << TOK_STR_ALLOC_INCR_BITS)
85#define TOK_MAX_SIZE 4 /* token max size in int unit when stored in string */
83 86
84/* token symbol management */87/* token symbol management */
85typedef struct TokenSym {88typedef struct TokenSym {
@@ -238,6 +241,7 @@ typedef struct ParseState { · 152 unmodified lines
238typedef struct TokenString {241typedef struct TokenString {
239 int *str;242 int *str;
240 int len;243 int len;
244 int allocated_len;
241 int last_line_num;245 int last_line_num;
242} TokenString;246} TokenString;
243 247
@@ -1115,26 +1119,11 @@ void test_lvalue(void) · 871 unmodified lines
1115 expect("lvalue");1119 expect("lvalue");
1116}1120}
1117 1121
1118TokenSym *tok_alloc(const char *str, int len)1122/* allocate a new token */
1123static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
1119{1124{
1120 TokenSym *ts, **pts, **ptable;1125 TokenSym *ts, **ptable;
1121 int i;1126 int i;
1122 unsigned int h;
1123
1124 h = 1;
1125 for(i=0;i<len;i++)
1126 h = h * 263 + ((unsigned char *)str)[i];
1127 h &= (TOK_HASH_SIZE - 1);
1128
1129 pts = &hash_ident[h];
1130 while (1) {
1131 ts = *pts;
1132 if (!ts)
1133 break;
1134 if (ts->len == len && !memcmp(ts->str, str, len))
1135 return ts;
1136 pts = &(ts->hash_next);
1137 }
1138 1127
1139 if (tok_ident >= SYM_FIRST_ANOM) 1128 if (tok_ident >= SYM_FIRST_ANOM)
1140 error("memory full");1129 error("memory full");
@@ -1163,6 +1152,33 @@ TokenSym *tok_alloc(const char *str, int len) · 22 unmodified lines
1163 return ts;1152 return ts;
1164}1153}
1165 1154
1155#define TOK_HASH_INIT 1
1156#define TOK_HASH_FUNC(h, c) ((h) * 263 + (c))
1157
1158/* find a token and add it if not found */
1159static TokenSym *tok_alloc(const char *str, int len)
1160{
1161 TokenSym *ts, **pts;
1162 int i;
1163 unsigned int h;
1164
1165 h = TOK_HASH_INIT;
1166 for(i=0;i<len;i++)
1167 h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
1168 h &= (TOK_HASH_SIZE - 1);
1169
1170 pts = &hash_ident[h];
1171 for(;;) {
1172 ts = *pts;
1173 if (!ts)
1174 break;
1175 if (ts->len == len && !memcmp(ts->str, str, len))
1176 return ts;
1177 pts = &(ts->hash_next);
1178 }
1179 return tok_alloc_new(pts, str, len);
1180}
1181
1166/* CString handling */1182/* CString handling */
1167 1183
1168static void cstr_realloc(CString *cstr, int new_size)1184static void cstr_realloc(CString *cstr, int new_size)
@@ -1904,6 +1920,7 @@ static inline void tok_str_new(TokenString *s) · 735 unmodified lines
1904{1920{
1905 s->str = NULL;1921 s->str = NULL;
1906 s->len = 0;1922 s->len = 0;
1923 s->allocated_len = 0;
1907 s->last_line_num = -1;1924 s->last_line_num = -1;
1908}1925}
1909 1926
@@ -1915,52 +1932,114 @@ static void tok_str_free(int *str) · 5 unmodified lines
1915 1932
1916 p = str;1933 p = str;
1917 for(;;) {1934 for(;;) {
1918 t = *p++;1935 t = *p;
1936 /* NOTE: we test zero separately so that GCC can generate a
1937 table for the following switch */
1919 if (t == 0)1938 if (t == 0)
1920 break;1939 break;
1921 if (t == TOK_STR || t == TOK_LSTR || t == TOK_PPNUM) {1940 switch(t) {
1941 case TOK_CINT:
1942 case TOK_CUINT:
1943 case TOK_CCHAR:
1944 case TOK_LCHAR:
1945 case TOK_CFLOAT:
1946 case TOK_LINENUM:
1947 p += 2;
1948 break;
1949 case TOK_PPNUM:
1950 case TOK_STR:
1951 case TOK_LSTR:
1922 /* XXX: use a macro to be portable on 64 bit ? */1952 /* XXX: use a macro to be portable on 64 bit ? */
1923 cstr = (CString *)(*p++);1953 cstr = (CString *)p[1];
1924 cstr_free(cstr);1954 cstr_free(cstr);
1925 tcc_free(cstr);1955 tcc_free(cstr);
1926 } else {1956 p += 2;
1927 p += tok_ext_size(t);1957 break;
1958 case TOK_CDOUBLE:
1959 case TOK_CLLONG:
1960 case TOK_CULLONG:
1961 p += 3;
1962 break;
1963 case TOK_CLDOUBLE:
1964 p += 1 + (LDOUBLE_SIZE / 4);
1965 break;
1966 default:
1967 p++;
1968 break;
1928 }1969 }
1929 }1970 }
1930 tcc_free(str);1971 tcc_free(str);
1931}1972}
1932 1973
1974static int *tok_str_realloc(TokenString *s)
1975{
1976 int *str, len;
1977
1978 len = s->allocated_len + TOK_STR_ALLOC_INCR;
1979 str = tcc_realloc(s->str, len * sizeof(int));
1980 if (!str)
1981 error("memory full");
1982 s->allocated_len = len;
1983 s->str = str;
1984 return str;
1985}
1986
1933static void tok_str_add(TokenString *s, int t)1987static void tok_str_add(TokenString *s, int t)
1934{1988{
1935 int len, *str;1989 int len, *str;
1936 1990
1937 len = s->len;1991 len = s->len;
1938 str = s->str;1992 str = s->str;
1939 if ((len & 63) == 0) {1993 if (len >= s->allocated_len)
1940 str = tcc_realloc(str, (len + 64) * sizeof(int));1994 str = tok_str_realloc(s);
1941 if (!str)
1942 return;
1943 s->str = str;
1944 }
1945 str[len++] = t;1995 str[len++] = t;
1946 s->len = len;1996 s->len = len;
1947}1997}
1948 1998
1949static void tok_str_add2(TokenString *s, int t, CValue *cv)1999static void tok_str_add2(TokenString *s, int t, CValue *cv)
1950{2000{
1951 int n, i;2001 int len, *str;
1952 CValue cv1;
1953 2002
1954 tok_str_add(s, t);2003 len = s->len;
1955 if (t == TOK_STR || t == TOK_LSTR || t == TOK_PPNUM) {2004 str = s->str;
1956 /* special case: need to duplicate string */2005
1957 cv1.cstr = cstr_dup(cv->cstr);2006 /* allocate space for worst case */
1958 tok_str_add(s, cv1.tab[0]);2007 if (len + TOK_MAX_SIZE > s->allocated_len)
1959 } else {2008 str = tok_str_realloc(s);
1960 n = tok_ext_size(t);2009 str[len++] = t;
1961 for(i=0;i<n;i++)2010 switch(t) {
1962 tok_str_add(s, cv->tab[i]);2011 case TOK_CINT:
2012 case TOK_CUINT:
2013 case TOK_CCHAR:
2014 case TOK_LCHAR:
2015 case TOK_CFLOAT:
2016 case TOK_LINENUM:
2017 str[len++] = cv->tab[0];
2018 break;
2019 case TOK_PPNUM:
2020 case TOK_STR:
2021 case TOK_LSTR:
2022 str[len++] = (int)cstr_dup(cv->cstr);
2023 break;
2024 case TOK_CDOUBLE:
2025 case TOK_CLLONG:
2026 case TOK_CULLONG:
2027 str[len++] = cv->tab[0];
2028 str[len++] = cv->tab[1];
2029 break;
2030 case TOK_CLDOUBLE:
2031#if LDOUBLE_SIZE == 12
2032 str[len++] = cv->tab[0];
2033 str[len++] = cv->tab[1];
2034 str[len++] = cv->tab[2];
2035#else
2036#error add long double size support
2037#endif
2038 break;
2039 default:
2040 break;
1963 }2041 }
2042 s->len = len;
1964}2043}
1965 2044
1966/* add the current parse token in token string 's' */2045/* add the current parse token in token string 's' */
@@ -1977,18 +2056,47 @@ static void tok_str_add_tok(TokenString *s) · 10 unmodified lines
1977 tok_str_add2(s, tok, &tokc);2056 tok_str_add2(s, tok, &tokc);
1978}2057}
1979 2058
1980/* get a token from an integer array and increment pointer accordingly */2059#if LDOUBLE_SIZE == 12
1981static int tok_get(int **tok_str, CValue *cv)2060#define LDOUBLE_GET(p, cv) \
1982{2061 cv.tab[0] = p[0]; \
1983 int *p, t, n, i;2062 cv.tab[1] = p[1]; \
2063 cv.tab[2] = p[2];
2064#else
2065#error add long double size support
2066#endif
2067
1984 2068
1985 p = *tok_str;2069/* get a token from an integer array and increment pointer
1986 t = *p++;2070 accordingly. we code it as a macro to avoid pointer aliasing. */
1987 n = tok_ext_size(t);2071#define TOK_GET(t, p, cv) \
1988 for(i=0;i<n;i++)2072{ \
1989 cv->tab[i] = *p++;2073 t = *p++; \
1990 *tok_str = p;2074 switch(t) { \
1991 return t;2075 case TOK_CINT: \
2076 case TOK_CUINT: \
2077 case TOK_CCHAR: \
2078 case TOK_LCHAR: \
2079 case TOK_CFLOAT: \
2080 case TOK_LINENUM: \
2081 case TOK_STR: \
2082 case TOK_LSTR: \
2083 case TOK_PPNUM: \
2084 cv.tab[0] = *p++; \
2085 break; \
2086 case TOK_CDOUBLE: \
2087 case TOK_CLLONG: \
2088 case TOK_CULLONG: \
2089 cv.tab[0] = p[0]; \
2090 cv.tab[1] = p[1]; \
2091 p += 2; \
2092 break; \
2093 case TOK_CLDOUBLE: \
2094 LDOUBLE_GET(p, cv); \
2095 p += LDOUBLE_SIZE / 4; \
2096 break; \
2097 default: \
2098 break; \
2099 } \
1992}2100}
1993 2101
1994/* defines handling */2102/* defines handling */
@@ -2059,7 +2167,7 @@ static Sym *label_push(int v, int flags) · 64 unmodified lines
2059}2167}
2060 2168
2061/* eval an expression for #if/#elif */2169/* eval an expression for #if/#elif */
2062int expr_preprocess(void)2170static int expr_preprocess(void)
2063{2171{
2064 int c, t;2172 int c, t;
2065 TokenString str;2173 TokenString str;
@@ -2096,13 +2204,13 @@ int expr_preprocess(void) · 30 unmodified lines
2096}2204}
2097 2205
2098#if defined(PARSE_DEBUG) || defined(PP_DEBUG)2206#if defined(PARSE_DEBUG) || defined(PP_DEBUG)
2099void tok_print(int *str)2207static void tok_print(int *str)
2100{2208{
2101 int t;2209 int t;
2102 CValue cval;2210 CValue cval;
2103 2211
2104 while (1) {2212 while (1) {
2105 t = tok_get(&str, &cval);2213 TOK_GET(t, str, cval);
2106 if (!t)2214 if (!t)
2107 break;2215 break;
2108 printf(" %s", get_tok_str(t, &cval));2216 printf(" %s", get_tok_str(t, &cval));
@@ -2876,6 +2984,7 @@ static inline void next_nomacro1(void) · 767 unmodified lines
2876 int b, t, c;2984 int b, t, c;
2877 TokenSym *ts;2985 TokenSym *ts;
2878 uint8_t *p, *p1;2986 uint8_t *p, *p1;
2987 unsigned int h;
2879 2988
2880 p = file->buf_ptr;2989 p = file->buf_ptr;
2881 redo_no_start:2990 redo_no_start:
@@ -2944,10 +3053,10 @@ static inline void next_nomacro1(void) · 62 unmodified lines
2944 break;3053 break;
2945 3054
2946 case '\n':3055 case '\n':
2947 file->line_num++;
2948 if (return_linefeed) {3056 if (return_linefeed) {
2949 tok = TOK_LINEFEED;3057 tok = TOK_LINEFEED;
2950 } else {3058 } else {
3059 file->line_num++;
2951 tok_flags |= TOK_FLAG_BOL;3060 tok_flags |= TOK_FLAG_BOL;
2952 p++;3061 p++;
2953 goto redo_no_start;3062 goto redo_no_start;
@@ -2989,16 +3098,35 @@ static inline void next_nomacro1(void) · 35 unmodified lines
2989 case '_':3098 case '_':
2990 parse_ident_fast:3099 parse_ident_fast:
2991 p1 = p;3100 p1 = p;
3101 h = TOK_HASH_INIT;
3102 h = TOK_HASH_FUNC(h, c);
2992 p++;3103 p++;
2993 for(;;) {3104 for(;;) {
2994 c = *p;3105 c = *p;
2995 if (!isid(c) && !isnum(c))3106 if (!isid(c) && !isnum(c))
2996 break;3107 break;
3108 h = TOK_HASH_FUNC(h, c);
2997 p++;3109 p++;
2998 }3110 }
2999 if (c != '\\') {3111 if (c != '\\') {
3000 /* fast case : no stray found, so we have the full token */3112 TokenSym **pts;
3001 ts = tok_alloc(p1, p - p1);3113 int len;
3114
3115 /* fast case : no stray found, so we have the full token
3116 and we have already hashed it */
3117 len = p - p1;
3118 h &= (TOK_HASH_SIZE - 1);
3119 pts = &hash_ident[h];
3120 for(;;) {
3121 ts = *pts;
3122 if (!ts)
3123 break;
3124 if (ts->len == len && !memcmp(ts->str, p1, len))
3125 goto token_found;
3126 pts = &(ts->hash_next);
3127 }
3128 ts = tok_alloc_new(pts, p1, len);
3129 token_found: ;
3002 } else {3130 } else {
3003 /* slower case */3131 /* slower case */
3004 cstr_reset(&tokcstr);3132 cstr_reset(&tokcstr);
@@ -3019,8 +3147,8 @@ static inline void next_nomacro1(void) · 14 unmodified lines
3019 tok = ts->tok;3147 tok = ts->tok;
3020 break;3148 break;
3021 case 'L':3149 case 'L':
3022 c = p[1];3150 t = p[1];
3023 if (c != '\\' && c != '\'' && c != '\"') {3151 if (t != '\\' && t != '\'' && t != '\"') {
3024 /* fast case */3152 /* fast case */
3025 goto parse_ident_fast;3153 goto parse_ident_fast;
3026 } else {3154 } else {
@@ -3268,7 +3396,7 @@ static void next_nomacro(void) · 241 unmodified lines
3268 redo:3396 redo:
3269 tok = *macro_ptr;3397 tok = *macro_ptr;
3270 if (tok) {3398 if (tok) {
3271 tok = tok_get(&macro_ptr, &tokc);3399 TOK_GET(tok, macro_ptr, tokc);
3272 if (tok == TOK_LINENUM) {3400 if (tok == TOK_LINENUM) {
3273 file->line_num = tokc.i;3401 file->line_num = tokc.i;
3274 goto redo;3402 goto redo;
@@ -3291,12 +3419,12 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 16 unmodified lines
3291 tok_str_new(&str);3419 tok_str_new(&str);
3292 last_tok = 0;3420 last_tok = 0;
3293 while(1) {3421 while(1) {
3294 t = tok_get(&macro_str, &cval);3422 TOK_GET(t, macro_str, cval);
3295 if (!t)3423 if (!t)
3296 break;3424 break;
3297 if (t == '#') {3425 if (t == '#') {
3298 /* stringize */3426 /* stringize */
3299 t = tok_get(&macro_str, &cval);3427 TOK_GET(t, macro_str, cval);
3300 if (!t)3428 if (!t)
3301 break;3429 break;
3302 s = sym_find2(args, t);3430 s = sym_find2(args, t);
@@ -3307,7 +3435,7 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 4 unmodified lines
3307 while (*st) {3435 while (*st) {
3308 if (notfirst)3436 if (notfirst)
3309 cstr_ccat(&cstr, ' ');3437 cstr_ccat(&cstr, ' ');
3310 t = tok_get(&st, &cval);3438 TOK_GET(t, st, cval);
3311 cstr_cat(&cstr, get_tok_str(t, &cval));3439 cstr_cat(&cstr, get_tok_str(t, &cval));
3312 notfirst = 1;3440 notfirst = 1;
3313 }3441 }
@@ -3348,7 +3476,7 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 34 unmodified lines
3348 int t1;3476 int t1;
3349 add_var:3477 add_var:
3350 for(;;) {3478 for(;;) {
3351 t1 = tok_get(&st, &cval);3479 TOK_GET(t1, st, cval);
3352 if (!t1)3480 if (!t1)
3353 break;3481 break;
3354 tok_str_add2(&str, t1, &cval);3482 tok_str_add2(&str, t1, &cval);
@@ -3392,7 +3520,7 @@ static int *macro_twosharps(void) · 37 unmodified lines
3392 macro_ptr1 = macro_ptr;3520 macro_ptr1 = macro_ptr;
3393 t = *macro_ptr;3521 t = *macro_ptr;
3394 if (t) {3522 if (t) {
3395 t = tok_get(&macro_ptr, &cval);3523 TOK_GET(t, macro_ptr, cval);
3396 3524
3397 /* We concatenate the two tokens if we have an3525 /* We concatenate the two tokens if we have an
3398 identifier or a preprocessing number */3526 identifier or a preprocessing number */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t