/* CPP Library - lexical analysis.   Copyright (C) 2000-2026 Free Software Foundation, Inc.   Contributed by Per Bothner, 1994-95.   Based on CCCP program by Paul Rubin, June 1986   Adapted to ANSI C, Richard Stallman, Jan 1987   Broken out to separate file, Zack Weinberg, Mar 2000 This program is free software; you can redistribute it and/or modify itunder the terms of the GNU General Public License as published by theFree Software Foundation; either version 3, or (at your option) anylater version. This program is distributed in the hope that it will be useful,but WITHOUT ANY WARRANTY; without even the implied warranty ofMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See theGNU General Public License for more details. You should have received a copy of the GNU General Public Licensealong with this program; see the file COPYING3.  If not see<http://www.gnu.org/licenses/>.  */ #include "config.h"#include "system.h"#include "cpplib.h"#include "internal.h" enum spell_type{  SPELL_OPERATOR = 0,  SPELL_IDENT,  SPELL_LITERAL,  SPELL_NONE}; struct token_spelling{  enum spell_type category;  const unsigned char *name;}; static const unsigned char *const digraph_spellings[] ={ UC"%:", UC"%:%:", UC"<:", UC":>", UC"<%", UC"%>" }; #define OP(e, s) { SPELL_OPERATOR, UC s  },#define TK(e, s) { SPELL_ ## s,    UC #e },static const struct token_spelling token_spellings[N_TTYPES] = { TTYPE_TABLE };#undef OP#undef TK #define TOKEN_SPELL(token) (token_spellings[(token)->type].category)#define TOKEN_NAME(token) (token_spellings[(token)->type].name) /* ISO 10646 defines the UCS codespace as the range 0-0x10FFFF inclusive.  */#define UCS_LIMIT 0x10FFFF static void add_line_note (cpp_buffer *, const uchar *, unsigned int);static int skip_line_comment (cpp_reader *);static void skip_whitespace (cpp_reader *, cppchar_t);static void lex_string (cpp_reader *, cpp_token *, const uchar *);static void save_comment (cpp_reader *, cpp_token *, const uchar *, cppchar_t);static void store_comment (cpp_reader *, cpp_token *);static void create_literal (cpp_reader *, cpp_token *, const uchar *,			    unsigned int, enum cpp_ttype);static bool warn_in_comment (cpp_reader *, _cpp_line_note *);static int name_p (cpp_reader *, const cpp_string *);static tokenrun *next_tokenrun (tokenrun *); static _cpp_buff *new_buff (size_t);  /* Utility routine:    Compares, the token TOKEN to the NUL-terminated string STRING.   TOKEN must be a CPP_NAME.  Returns 1 for equal, 0 for unequal.  */intcpp_ideq (const cpp_token *token, const char *string){  if (token->type != CPP_NAME)    return 0;   return !ustrcmp (NODE_NAME (token->val.node.node), (const uchar *) string);} /* Record a note TYPE at byte POS into the current cleaned logical   line.  */static voidadd_line_note (cpp_buffer *buffer, const uchar *pos, unsigned int type){  if (buffer->notes_used == buffer->notes_cap)    {      buffer->notes_cap = buffer->notes_cap * 2 + 200;      buffer->notes = XRESIZEVEC (_cpp_line_note, buffer->notes,                                  buffer->notes_cap);    }   buffer->notes[buffer->notes_used].pos = pos;  buffer->notes[buffer->notes_used].type = type;  buffer->notes_used++;} /* Fast path to find line special characters using optimized character   scanning algorithms.  Anything complicated falls back to the slow   path below.  Since this loop is very hot it's worth doing these kinds   of optimizations.    One of the paths through the ifdefs should provide      const uchar *search_line_fast (const uchar *s, const uchar *end);    Between S and END, search for \n, \r, \\, ?.  Return a pointer to   the found character.    Note that the last character of the buffer is *always* a newline,   as forced by _cpp_convert_input.  This fact can be used to avoid   explicitly looking for the end of the buffer.  */ /* Configure gives us an ifdef test.  */#ifndef WORDS_BIGENDIAN#define WORDS_BIGENDIAN 0#endif /* We'd like the largest integer that fits into a register.  There's nothing   in <stdint.h> that gives us that.  For most hosts this is unsigned long,   but MS decided on an LLP64 model.  Thankfully when building with GCC we   can get the "real" word size.  */#ifdef __GNUC__typedef unsigned int word_type __attribute__((__mode__(__word__)));#elsetypedef unsigned long word_type;#endif /* The code below is only expecting sizes 4 or 8.   Die at compile-time if this expectation is violated.  */typedef char check_word_type_size  [(sizeof(word_type) == 8 || sizeof(word_type) == 4) * 2 - 1]; /* Return X with the first N bytes forced to values that won't match one   of the interesting characters.  Note that NUL is not interesting.  */ static inline word_typeacc_char_mask_misalign (word_type val, unsigned int n){  word_type mask = -1;  if (WORDS_BIGENDIAN)    mask >>= n * 8;  else    mask <<= n * 8;  return val & mask;} /* Return X replicated to all byte positions within WORD_TYPE.  */ static inline word_typeacc_char_replicate (uchar x){  word_type ret;   ret = (x << 24) | (x << 16) | (x << 8) | x;  if (sizeof(word_type) == 8)    ret = (ret << 16 << 16) | ret;  return ret;} /* Return non-zero if some byte of VAL is (probably) C.  */ static inline word_typeacc_char_cmp (word_type val, word_type c){#if defined(__GNUC__) && defined(__alpha__)  /* We can get exact results using a compare-bytes instruction.     Get (val == c) via (0 >= (val ^ c)).  */  return __builtin_alpha_cmpbge (0, val ^ c);#else  word_type magic = 0x7efefefeU;  if (sizeof(word_type) == 8)    magic = (magic << 16 << 16) | 0xfefefefeU;  magic |= 1;   val ^= c;  return ((val + magic) ^ ~val) & ~magic;#endif} /* Given the result of acc_char_cmp is non-zero, return the index of   the found character.  If this was a false positive, return -1.  */ static inline intacc_char_index (word_type cmp ATTRIBUTE_UNUSED,		word_type val ATTRIBUTE_UNUSED){#if defined(__GNUC__) && defined(__alpha__) && !WORDS_BIGENDIAN  /* The cmpbge instruction sets *bits* of the result corresponding to     matches in the bytes with no false positives.  */  return __builtin_ctzl (cmp);#else  unsigned int i;   /* ??? It would be nice to force unrolling here,     and have all of these constants folded.  */  for (i = 0; i < sizeof(word_type); ++i)    {      uchar c;      if (WORDS_BIGENDIAN)	c = (val >> (sizeof(word_type) - i - 1) * 8) & 0xff;      else	c = (val >> i * 8) & 0xff;       if (c == '\n' || c == '\r' || c == '\\' || c == '?')	return i;    }   return -1;#endif} /* A version of the fast scanner using bit fiddling techniques.    For 32-bit words, one would normally perform 16 comparisons and   16 branches.  With this algorithm one performs 24 arithmetic   operations and one branch.  Whether this is faster with a 32-bit   word size is going to be somewhat system dependent.    For 64-bit words, we eliminate twice the number of comparisons   and branches without increasing the number of arithmetic operations.   It's almost certainly going to be a win with 64-bit word size.  */ static inline const uchar *search_line_acc_char (const uchar *s, const uchar *end ATTRIBUTE_UNUSED){  const word_type repl_nl = acc_char_replicate ('\n');  const word_type repl_cr = acc_char_replicate ('\r');  const word_type repl_bs = acc_char_replicate ('\\');  const word_type repl_qm = acc_char_replicate ('?');   unsigned int misalign;  const word_type *p;  word_type val, t;   /* Align the buffer.  Mask out any bytes from before the beginning.  */  p = (word_type *)((uintptr_t)s & -sizeof(word_type));  val = *p;  misalign = (uintptr_t)s & (sizeof(word_type) - 1);  if (misalign)    val = acc_char_mask_misalign (val, misalign);   /* Main loop.  */  while (1)    {      t  = acc_char_cmp (val, repl_nl);      t |= acc_char_cmp (val, repl_cr);      t |= acc_char_cmp (val, repl_bs);      t |= acc_char_cmp (val, repl_qm);       if (__builtin_expect (t != 0, 0))	{	  int i = acc_char_index (t, val);	  if (i >= 0)	    return (const uchar *)p + i;	}       val = *++p;    }} /* Disable on Solaris 2/x86 until the following problem can be properly   autoconfed:    The Solaris 10+ assembler tags objects with the instruction set   extensions used, so SSE4.2 executables cannot run on machines that   don't support that extension.  */ #if (GCC_VERSION >= 4005) && (__GNUC__ >= 5 || !defined(__PIC__)) && (defined(__i386__) || defined(__x86_64__)) && !(defined(__sun__) && defined(__svr4__)) /* Replicated character data to be shared between implementations.   Recall that outside of a context with vector support we can't   define compatible vector types, therefore these are all defined   in terms of raw characters.  */static const char repl_chars[4][16] __attribute__((aligned(16))) = {  { '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',    '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n' },  { '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',    '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r' },  { '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',    '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\' },  { '?', '?', '?', '?', '?', '?', '?', '?',    '?', '?', '?', '?', '?', '?', '?', '?' },};  /* A version of the fast scanner using SSE2 vectorized byte compare insns.  */ static inline const uchar *#ifndef __SSE2____attribute__((__target__("sse2")))#endifsearch_line_sse2 (const uchar *s, const uchar *end ATTRIBUTE_UNUSED){  typedef char v16qi __attribute__ ((__vector_size__ (16)));   const v16qi repl_nl = *(const v16qi *)repl_chars[0];  const v16qi repl_cr = *(const v16qi *)repl_chars[1];  const v16qi repl_bs = *(const v16qi *)repl_chars[2];  const v16qi repl_qm = *(const v16qi *)repl_chars[3];   unsigned int misalign, found, mask;  const v16qi *p;  v16qi data, t;   /* Align the source pointer.  */  misalign = (uintptr_t)s & 15;  p = (const v16qi *)((uintptr_t)s & -16);  data = *p;   /* Create a mask for the bytes that are valid within the first     16-byte block.  The Idea here is that the AND with the mask     within the loop is "free", since we need some AND or TEST     insn in order to set the flags for the branch anyway.  */  mask = -1u << misalign;   /* Main loop processing 16 bytes at a time.  */  goto start;  do    {      data = *++p;      mask = -1;     start:      t  = data == repl_nl;      t |= data == repl_cr;      t |= data == repl_bs;      t |= data == repl_qm;      found = __builtin_ia32_pmovmskb128 (t);      found &= mask;    }  while (!found);   /* FOUND contains 1 in bits for which we matched a relevant     character.  Conversion to the byte index is trivial.  */  found = __builtin_ctz(found);  return (const uchar *)p + found;} #ifdef HAVE_SSSE3/* A version of the fast scanner using SSSE3 shuffle (PSHUFB) insns.  */ static inline const uchar *#ifndef __SSSE3____attribute__((__target__("ssse3")))#endifsearch_line_ssse3 (const uchar *s, const uchar *end ATTRIBUTE_UNUSED){  typedef char v16qi __attribute__ ((__vector_size__ (16)));  typedef v16qi v16qi_u __attribute__ ((__aligned__ (1)));  /* Helper vector for pshufb-based matching:     each character C we're searching for is at position (C % 16).  */  v16qi lut = { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, '\n', 0, '\\', '\r', 0, '?' };  static_assert('\n' == 10 && '\r' == 13 && '\\' == 92 && '?' == 63,                "host character encoding is ASCII");   v16qi d1, d2, t1, t2;  /* Unaligned loads, potentially using padding after the final newline.  */  static_assert (CPP_BUFFER_PADDING >= 64, "");  d1 = *(const v16qi_u *)s;  d2 = *(const v16qi_u *)(s + 16);  unsigned m1, m2, found;  /* Process two 16-byte chunks per iteration.  */  do    {      t1 = __builtin_ia32_pshufb128 (lut, d1);      t2 = __builtin_ia32_pshufb128 (lut, d2);      m1 = __builtin_ia32_pmovmskb128 (t1 == d1);      m2 = __builtin_ia32_pmovmskb128 (t2 == d2);      s += 32;      d1 = *(const v16qi_u *)s;      d2 = *(const v16qi_u *)(s + 16);      found = m1 + (m2 << 16);    }  while (!found);  /* Prefer to compute 's - 32' here, not spend an extra instruction     to make a copy of the previous value of 's' in the loop.  */  __asm__ ("" : "+r"(s));  return s - 32 + __builtin_ctz (found);} #else/* Work around out-dated assemblers without SSSE3 support.  */#define search_line_ssse3 search_line_sse2#endif #ifdef __SSSE3__/* No need for CPU probing, just use the best available variant.  */#define search_line_fast search_line_ssse3#else/* Check the CPU capabilities.  */ #include "../gcc/config/i386/cpuid.h" typedef const uchar * (*search_line_fast_type) (const uchar *, const uchar *);static search_line_fast_type search_line_fast#if defined(__SSE2__) = search_line_sse2;#else = search_line_acc_char;#endif #define HAVE_init_vectorized_lexer 1static inline voidinit_vectorized_lexer (void){  unsigned ax, bx, cx, dx;   if (!__get_cpuid (1, &ax, &bx, &cx, &dx))    return;   if (cx & bit_SSSE3)    search_line_fast = search_line_ssse3;  else if (dx & bit_SSE2)    search_line_fast = search_line_sse2;}#endif #elif (GCC_VERSION >= 4005) && defined(_ARCH_PWR8) && defined(__ALTIVEC__) /* A vection of the fast scanner using AltiVec vectorized byte compares   and VSX unaligned loads (when VSX is available).  This is otherwise   the same as the AltiVec version.  */ ATTRIBUTE_NO_SANITIZE_UNDEFINEDstatic const uchar *search_line_fast (const uchar *s, const uchar *end ATTRIBUTE_UNUSED){  typedef __attribute__((altivec(vector))) unsigned char vc;   const vc repl_nl = {    '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n',    '\n', '\n', '\n', '\n', '\n', '\n', '\n', '\n'  };  const vc repl_cr = {    '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r',    '\r', '\r', '\r', '\r', '\r', '\r', '\r', '\r'  };  const vc repl_bs = {    '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\',    '\\', '\\', '\\', '\\', '\\', '\\', '\\', '\\'  };  const vc repl_qm = {    '?', '?', '?', '?', '?', '?', '?', '?',    '?', '?', '?', '?', '?', '?', '?', '?',  };  const vc zero = { 0 };   vc data, t;   /* Main loop processing 16 bytes at a time.  */  do    {      vc m_nl, m_cr, m_bs, m_qm;       data = __builtin_vec_vsx_ld (0, s);      s += 16;       m_nl = (vc) __builtin_vec_cmpeq(data, repl_nl);      m_cr = (vc) __builtin_vec_cmpeq(data, repl_cr);      m_bs = (vc) __builtin_vec_cmpeq(data, repl_bs);      m_qm = (vc) __builtin_vec_cmpeq(data, repl_qm);      t = (m_nl | m_cr) | (m_bs | m_qm);       /* T now contains 0xff in bytes for which we matched one of the relevant	 characters.  We want to exit the loop if any byte in T is non-zero.	 Below is the expansion of vec_any_ne(t, zero).  */    }  while (!__builtin_vec_vcmpeq_p(/*__CR6_LT_REV*/3, t, zero));   /* Restore s to to point to the 16 bytes we just processed.  */  s -= 16;   {#define N  (sizeof(vc) / sizeof(long))     union {      vc v;      /* Statically assert that N is 2 or 4.  */      unsigned long l[(N == 2 || N == 4) ? N : -1];    } u;    unsigned long l, i = 0;     u.v = t;     /* Find the first word of T that is non-zero.  */    switch (N)      {      case 4:	l = u.l[i++];	if (l != 0)	  break;	s += sizeof(unsigned long);	l = u.l[i++];	if (l != 0)	  break;

Lines 1–500 of 5854. Load more