History

optimized token strings - optimized token hashing

bellard committed 23 years ago

Commit 9e180471 parent 8901fbe1 file · +193 −65Browse files

Unified · Split

tcc.c+193−65
@@ -80,6 +80,9 @@ · 79 unmodified lines
8080
8181 #define TOK_HASH_SIZE 2048 /* must be a power of two */
8282 #define TOK_ALLOC_INCR 512 /* must be a power of two */
83+#define TOK_STR_ALLOC_INCR_BITS 6
84+#define TOK_STR_ALLOC_INCR (1 << TOK_STR_ALLOC_INCR_BITS)
85+#define TOK_MAX_SIZE 4 /* token max size in int unit when stored in string */
8386
8487 /* token symbol management */
8588 typedef struct TokenSym {
@@ -238,6 +241,7 @@ typedef struct ParseState { · 152 unmodified lines
238241 typedef struct TokenString {
239242 int *str;
240243 int len;
244+ int allocated_len;
241245 int last_line_num;
242246 } TokenString;
243247
@@ -1115,26 +1119,11 @@ void test_lvalue(void) · 871 unmodified lines
11151119 expect("lvalue");
11161120 }
11171121
1118−TokenSym *tok_alloc(const char *str, int len)
1122+/* allocate a new token */
1123+static TokenSym *tok_alloc_new(TokenSym **pts, const char *str, int len)
11191124 {
1120− TokenSym *ts, **pts, **ptable;
1125+ TokenSym *ts, **ptable;
11211126 int i;
1122− unsigned int h;
1123−
1124− h = 1;
1125− for(i=0;i<len;i++)
1126− h = h * 263 + ((unsigned char *)str)[i];
1127− h &= (TOK_HASH_SIZE - 1);
1128−
1129− pts = &hash_ident[h];
1130− while (1) {
1131− ts = *pts;
1132− if (!ts)
1133− break;
1134− if (ts->len == len && !memcmp(ts->str, str, len))
1135− return ts;
1136− pts = &(ts->hash_next);
1137− }
11381127
11391128 if (tok_ident >= SYM_FIRST_ANOM)
11401129 error("memory full");
@@ -1163,6 +1152,33 @@ TokenSym *tok_alloc(const char *str, int len) · 22 unmodified lines
11631152 return ts;
11641153 }
11651154
1155+#define TOK_HASH_INIT 1
1156+#define TOK_HASH_FUNC(h, c) ((h) * 263 + (c))
1157+
1158+/* find a token and add it if not found */
1159+static TokenSym *tok_alloc(const char *str, int len)
1160+{
1161+ TokenSym *ts, **pts;
1162+ int i;
1163+ unsigned int h;
1164+
1165+ h = TOK_HASH_INIT;
1166+ for(i=0;i<len;i++)
1167+ h = TOK_HASH_FUNC(h, ((unsigned char *)str)[i]);
1168+ h &= (TOK_HASH_SIZE - 1);
1169+
1170+ pts = &hash_ident[h];
1171+ for(;;) {
1172+ ts = *pts;
1173+ if (!ts)
1174+ break;
1175+ if (ts->len == len && !memcmp(ts->str, str, len))
1176+ return ts;
1177+ pts = &(ts->hash_next);
1178+ }
1179+ return tok_alloc_new(pts, str, len);
1180+}
1181+
11661182 /* CString handling */
11671183
11681184 static void cstr_realloc(CString *cstr, int new_size)
@@ -1904,6 +1920,7 @@ static inline void tok_str_new(TokenString *s) · 735 unmodified lines
19041920 {
19051921 s->str = NULL;
19061922 s->len = 0;
1923+ s->allocated_len = 0;
19071924 s->last_line_num = -1;
19081925 }
19091926
@@ -1915,52 +1932,114 @@ static void tok_str_free(int *str) · 5 unmodified lines
19151932
19161933 p = str;
19171934 for(;;) {
1918− t = *p++;
1935+ t = *p;
1936+ /* NOTE: we test zero separately so that GCC can generate a
1937+ table for the following switch */
19191938 if (t == 0)
19201939 break;
1921− if (t == TOK_STR || t == TOK_LSTR || t == TOK_PPNUM) {
1940+ switch(t) {
1941+ case TOK_CINT:
1942+ case TOK_CUINT:
1943+ case TOK_CCHAR:
1944+ case TOK_LCHAR:
1945+ case TOK_CFLOAT:
1946+ case TOK_LINENUM:
1947+ p += 2;
1948+ break;
1949+ case TOK_PPNUM:
1950+ case TOK_STR:
1951+ case TOK_LSTR:
19221952 /* XXX: use a macro to be portable on 64 bit ? */
1923− cstr = (CString *)(*p++);
1953+ cstr = (CString *)p[1];
19241954 cstr_free(cstr);
19251955 tcc_free(cstr);
1926− } else {
1927− p += tok_ext_size(t);
1956+ p += 2;
1957+ break;
1958+ case TOK_CDOUBLE:
1959+ case TOK_CLLONG:
1960+ case TOK_CULLONG:
1961+ p += 3;
1962+ break;
1963+ case TOK_CLDOUBLE:
1964+ p += 1 + (LDOUBLE_SIZE / 4);
1965+ break;
1966+ default:
1967+ p++;
1968+ break;
19281969 }
19291970 }
19301971 tcc_free(str);
19311972 }
19321973
1974+static int *tok_str_realloc(TokenString *s)
1975+{
1976+ int *str, len;
1977+
1978+ len = s->allocated_len + TOK_STR_ALLOC_INCR;
1979+ str = tcc_realloc(s->str, len * sizeof(int));
1980+ if (!str)
1981+ error("memory full");
1982+ s->allocated_len = len;
1983+ s->str = str;
1984+ return str;
1985+}
1986+
19331987 static void tok_str_add(TokenString *s, int t)
19341988 {
19351989 int len, *str;
19361990
19371991 len = s->len;
19381992 str = s->str;
1939− if ((len & 63) == 0) {
1940− str = tcc_realloc(str, (len + 64) * sizeof(int));
1941− if (!str)
1942− return;
1943− s->str = str;
1944− }
1993+ if (len >= s->allocated_len)
1994+ str = tok_str_realloc(s);
19451995 str[len++] = t;
19461996 s->len = len;
19471997 }
19481998
19491999 static void tok_str_add2(TokenString *s, int t, CValue *cv)
19502000 {
1951− int n, i;
1952− CValue cv1;
2001+ int len, *str;
19532002
1954− tok_str_add(s, t);
1955− if (t == TOK_STR || t == TOK_LSTR || t == TOK_PPNUM) {
1956− /* special case: need to duplicate string */
1957− cv1.cstr = cstr_dup(cv->cstr);
1958− tok_str_add(s, cv1.tab[0]);
1959− } else {
1960− n = tok_ext_size(t);
1961− for(i=0;i<n;i++)
1962− tok_str_add(s, cv->tab[i]);
2003+ len = s->len;
2004+ str = s->str;
2005+
2006+ /* allocate space for worst case */
2007+ if (len + TOK_MAX_SIZE > s->allocated_len)
2008+ str = tok_str_realloc(s);
2009+ str[len++] = t;
2010+ switch(t) {
2011+ case TOK_CINT:
2012+ case TOK_CUINT:
2013+ case TOK_CCHAR:
2014+ case TOK_LCHAR:
2015+ case TOK_CFLOAT:
2016+ case TOK_LINENUM:
2017+ str[len++] = cv->tab[0];
2018+ break;
2019+ case TOK_PPNUM:
2020+ case TOK_STR:
2021+ case TOK_LSTR:
2022+ str[len++] = (int)cstr_dup(cv->cstr);
2023+ break;
2024+ case TOK_CDOUBLE:
2025+ case TOK_CLLONG:
2026+ case TOK_CULLONG:
2027+ str[len++] = cv->tab[0];
2028+ str[len++] = cv->tab[1];
2029+ break;
2030+ case TOK_CLDOUBLE:
2031+#if LDOUBLE_SIZE == 12
2032+ str[len++] = cv->tab[0];
2033+ str[len++] = cv->tab[1];
2034+ str[len++] = cv->tab[2];
2035+#else
2036+#error add long double size support
2037+#endif
2038+ break;
2039+ default:
2040+ break;
19632041 }
2042+ s->len = len;
19642043 }
19652044
19662045 /* add the current parse token in token string 's' */
@@ -1977,18 +2056,47 @@ static void tok_str_add_tok(TokenString *s) · 10 unmodified lines
19772056 tok_str_add2(s, tok, &tokc);
19782057 }
19792058
1980−/* get a token from an integer array and increment pointer accordingly */
1981−static int tok_get(int **tok_str, CValue *cv)
1982−{
1983− int *p, t, n, i;
2059+#if LDOUBLE_SIZE == 12
2060+#define LDOUBLE_GET(p, cv) \
2061+ cv.tab[0] = p[0]; \
2062+ cv.tab[1] = p[1]; \
2063+ cv.tab[2] = p[2];
2064+#else
2065+#error add long double size support
2066+#endif
2067+
19842068
1985− p = *tok_str;
1986− t = *p++;
1987− n = tok_ext_size(t);
1988− for(i=0;i<n;i++)
1989− cv->tab[i] = *p++;
1990− *tok_str = p;
1991− return t;
2069+/* get a token from an integer array and increment pointer
2070+ accordingly. we code it as a macro to avoid pointer aliasing. */
2071+#define TOK_GET(t, p, cv) \
2072+{ \
2073+ t = *p++; \
2074+ switch(t) { \
2075+ case TOK_CINT: \
2076+ case TOK_CUINT: \
2077+ case TOK_CCHAR: \
2078+ case TOK_LCHAR: \
2079+ case TOK_CFLOAT: \
2080+ case TOK_LINENUM: \
2081+ case TOK_STR: \
2082+ case TOK_LSTR: \
2083+ case TOK_PPNUM: \
2084+ cv.tab[0] = *p++; \
2085+ break; \
2086+ case TOK_CDOUBLE: \
2087+ case TOK_CLLONG: \
2088+ case TOK_CULLONG: \
2089+ cv.tab[0] = p[0]; \
2090+ cv.tab[1] = p[1]; \
2091+ p += 2; \
2092+ break; \
2093+ case TOK_CLDOUBLE: \
2094+ LDOUBLE_GET(p, cv); \
2095+ p += LDOUBLE_SIZE / 4; \
2096+ break; \
2097+ default: \
2098+ break; \
2099+ } \
19922100 }
19932101
19942102 /* defines handling */
@@ -2059,7 +2167,7 @@ static Sym *label_push(int v, int flags) · 64 unmodified lines
20592167 }
20602168
20612169 /* eval an expression for #if/#elif */
2062−int expr_preprocess(void)
2170+static int expr_preprocess(void)
20632171 {
20642172 int c, t;
20652173 TokenString str;
@@ -2096,13 +2204,13 @@ int expr_preprocess(void) · 30 unmodified lines
20962204 }
20972205
20982206 #if defined(PARSE_DEBUG) || defined(PP_DEBUG)
2099−void tok_print(int *str)
2207+static void tok_print(int *str)
21002208 {
21012209 int t;
21022210 CValue cval;
21032211
21042212 while (1) {
2105− t = tok_get(&str, &cval);
2213+ TOK_GET(t, str, cval);
21062214 if (!t)
21072215 break;
21082216 printf(" %s", get_tok_str(t, &cval));
@@ -2876,6 +2984,7 @@ static inline void next_nomacro1(void) · 767 unmodified lines
28762984 int b, t, c;
28772985 TokenSym *ts;
28782986 uint8_t *p, *p1;
2987+ unsigned int h;
28792988
28802989 p = file->buf_ptr;
28812990 redo_no_start:
@@ -2944,10 +3053,10 @@ static inline void next_nomacro1(void) · 62 unmodified lines
29443053 break;
29453054
29463055 case '\n':
2947− file->line_num++;
29483056 if (return_linefeed) {
29493057 tok = TOK_LINEFEED;
29503058 } else {
3059+ file->line_num++;
29513060 tok_flags |= TOK_FLAG_BOL;
29523061 p++;
29533062 goto redo_no_start;
@@ -2989,16 +3098,35 @@ static inline void next_nomacro1(void) · 35 unmodified lines
29893098 case '_':
29903099 parse_ident_fast:
29913100 p1 = p;
3101+ h = TOK_HASH_INIT;
3102+ h = TOK_HASH_FUNC(h, c);
29923103 p++;
29933104 for(;;) {
29943105 c = *p;
29953106 if (!isid(c) && !isnum(c))
29963107 break;
3108+ h = TOK_HASH_FUNC(h, c);
29973109 p++;
29983110 }
29993111 if (c != '\\') {
3000− /* fast case : no stray found, so we have the full token */
3001− ts = tok_alloc(p1, p - p1);
3112+ TokenSym **pts;
3113+ int len;
3114+
3115+ /* fast case : no stray found, so we have the full token
3116+ and we have already hashed it */
3117+ len = p - p1;
3118+ h &= (TOK_HASH_SIZE - 1);
3119+ pts = &hash_ident[h];
3120+ for(;;) {
3121+ ts = *pts;
3122+ if (!ts)
3123+ break;
3124+ if (ts->len == len && !memcmp(ts->str, p1, len))
3125+ goto token_found;
3126+ pts = &(ts->hash_next);
3127+ }
3128+ ts = tok_alloc_new(pts, p1, len);
3129+ token_found: ;
30023130 } else {
30033131 /* slower case */
30043132 cstr_reset(&tokcstr);
@@ -3019,8 +3147,8 @@ static inline void next_nomacro1(void) · 14 unmodified lines
30193147 tok = ts->tok;
30203148 break;
30213149 case 'L':
3022− c = p[1];
3023− if (c != '\\' && c != '\'' && c != '\"') {
3150+ t = p[1];
3151+ if (t != '\\' && t != '\'' && t != '\"') {
30243152 /* fast case */
30253153 goto parse_ident_fast;
30263154 } else {
@@ -3268,7 +3396,7 @@ static void next_nomacro(void) · 241 unmodified lines
32683396 redo:
32693397 tok = *macro_ptr;
32703398 if (tok) {
3271− tok = tok_get(&macro_ptr, &tokc);
3399+ TOK_GET(tok, macro_ptr, tokc);
32723400 if (tok == TOK_LINENUM) {
32733401 file->line_num = tokc.i;
32743402 goto redo;
@@ -3291,12 +3419,12 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 16 unmodified lines
32913419 tok_str_new(&str);
32923420 last_tok = 0;
32933421 while(1) {
3294− t = tok_get(&macro_str, &cval);
3422+ TOK_GET(t, macro_str, cval);
32953423 if (!t)
32963424 break;
32973425 if (t == '#') {
32983426 /* stringize */
3299− t = tok_get(&macro_str, &cval);
3427+ TOK_GET(t, macro_str, cval);
33003428 if (!t)
33013429 break;
33023430 s = sym_find2(args, t);
@@ -3307,7 +3435,7 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 4 unmodified lines
33073435 while (*st) {
33083436 if (notfirst)
33093437 cstr_ccat(&cstr, ' ');
3310− t = tok_get(&st, &cval);
3438+ TOK_GET(t, st, cval);
33113439 cstr_cat(&cstr, get_tok_str(t, &cval));
33123440 notfirst = 1;
33133441 }
@@ -3348,7 +3476,7 @@ static int *macro_arg_subst(Sym **nested_list, int *macro_str, Sym *args) · 34 unmodified lines
33483476 int t1;
33493477 add_var:
33503478 for(;;) {
3351− t1 = tok_get(&st, &cval);
3479+ TOK_GET(t1, st, cval);
33523480 if (!t1)
33533481 break;
33543482 tok_str_add2(&str, t1, &cval);
@@ -3392,7 +3520,7 @@ static int *macro_twosharps(void) · 37 unmodified lines
33923520 macro_ptr1 = macro_ptr;
33933521 t = *macro_ptr;
33943522 if (t) {
3395− t = tok_get(&macro_ptr, &cval);
3523+ TOK_GET(t, macro_ptr, cval);
33963524
33973525 /* We concatenate the two tokens if we have an
33983526 identifier or a preprocessing number */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t