/* Tokenizer implementation */ #include "Python.h"#include "pgenheaders.h" #include <ctype.h>#include <assert.h> #include "tokenizer.h"#include "errcode.h" #ifndef PGEN#include "unicodeobject.h"#include "stringobject.h"#include "fileobject.h"#include "codecs.h"#include "abstract.h"#endif /* PGEN */ extern char *PyOS_Readline(FILE *, FILE *, char *);/* Return malloc'ed string including trailing \n;   empty malloc'ed string for EOF;   NULL if interrupted */ /* Don't ever change this -- it would break the portability of Python code */#define TABSIZE 8 /* Convert a possibly signed character to a nonnegative int *//* XXX This assumes characters are 8 bits wide */#ifdef __CHAR_UNSIGNED__#define Py_CHARMASK(c)		(c)#else#define Py_CHARMASK(c)		((c) & 0xff)#endif /* Forward */static struct tok_state *tok_new(void);static int tok_nextc(struct tok_state *tok);static void tok_backup(struct tok_state *tok, int c); /* Token names */ char *_PyParser_TokenNames[] = {	"ENDMARKER",	"NAME",	"NUMBER",	"STRING",	"NEWLINE",	"INDENT",	"DEDENT",	"LPAR",	"RPAR",	"LSQB",	"RSQB",	"COLON",	"COMMA",	"SEMI",	"PLUS",	"MINUS",	"STAR",	"SLASH",	"VBAR",	"AMPER",	"LESS",	"GREATER",	"EQUAL",	"DOT",	"PERCENT",	"BACKQUOTE",	"LBRACE",	"RBRACE",	"EQEQUAL",	"NOTEQUAL",	"LESSEQUAL",	"GREATEREQUAL",	"TILDE",	"CIRCUMFLEX",	"LEFTSHIFT",	"RIGHTSHIFT",	"DOUBLESTAR",	"PLUSEQUAL",	"MINEQUAL",	"STAREQUAL",	"SLASHEQUAL",	"PERCENTEQUAL",	"AMPEREQUAL",	"VBAREQUAL",	"CIRCUMFLEXEQUAL",	"LEFTSHIFTEQUAL",	"RIGHTSHIFTEQUAL",	"DOUBLESTAREQUAL",	"DOUBLESLASH",	"DOUBLESLASHEQUAL",	"AT",	/* This table must match the #defines in token.h! */	"OP",	"<ERRORTOKEN>",	"<N_TOKENS>"};  /* Create and initialize a new tok_state structure */ static struct tok_state *tok_new(void){	struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(                                                sizeof(struct tok_state));	if (tok == NULL)		return NULL;	tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;	tok->done = E_OK;	tok->fp = NULL;	tok->tabsize = TABSIZE;	tok->indent = 0;	tok->indstack[0] = 0;	tok->atbol = 1;	tok->pendin = 0;	tok->prompt = tok->nextprompt = NULL;	tok->lineno = 0;	tok->level = 0;	tok->filename = NULL;	tok->altwarning = 0;	tok->alterror = 0;	tok->alttabsize = 1;	tok->altindstack[0] = 0;	tok->decoding_state = 0;	tok->decoding_erred = 0;	tok->read_coding_spec = 0;	tok->encoding = NULL;        tok->cont_line = 0;#ifndef PGEN	tok->decoding_readline = NULL;	tok->decoding_buffer = NULL;#endif	return tok;} #ifdef PGEN static char *decoding_fgets(char *s, int size, struct tok_state *tok){	return fgets(s, size, tok->fp);} static intdecoding_feof(struct tok_state *tok){	return feof(tok->fp);} static const char *decode_str(const char *str, struct tok_state *tok){	return str;} #else /* PGEN */ static char *error_ret(struct tok_state *tok) /* XXX */{	tok->decoding_erred = 1;	if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */		PyMem_FREE(tok->buf);	tok->buf = NULL;	return NULL;		/* as if it were EOF */} static char *new_string(const char *s, Py_ssize_t len){	char* result = (char *)PyMem_MALLOC(len + 1);	if (result != NULL) {		memcpy(result, s, len);		result[len] = '\0';	}	return result;} static char *get_normal_name(char *s)	/* for utf-8 and latin-1 */{	char buf[13];	int i;	for (i = 0; i < 12; i++) {		int c = s[i];		if (c == '\0') break;		else if (c == '_') buf[i] = '-';		else buf[i] = tolower(c);	}	buf[i] = '\0';	if (strcmp(buf, "utf-8") == 0 ||	    strncmp(buf, "utf-8-", 6) == 0) return "utf-8";	else if (strcmp(buf, "latin-1") == 0 ||		 strcmp(buf, "iso-8859-1") == 0 ||		 strcmp(buf, "iso-latin-1") == 0 ||		 strncmp(buf, "latin-1-", 8) == 0 ||		 strncmp(buf, "iso-8859-1-", 11) == 0 ||		 strncmp(buf, "iso-latin-1-", 12) == 0) return "iso-8859-1";	else return s;} /* Return the coding spec in S, or NULL if none is found.  */ static char *get_coding_spec(const char *s, Py_ssize_t size){	Py_ssize_t i;	/* Coding spec must be in a comment, and that comment must be         * the only statement on the source code line. */        for (i = 0; i < size - 6; i++) {		if (s[i] == '#')			break;		if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014')			return NULL;	}	for (; i < size - 6; i++) { /* XXX inefficient search */		const char* t = s + i;		if (strncmp(t, "coding", 6) == 0) {			const char* begin = NULL;			t += 6;			if (t[0] != ':' && t[0] != '=')				continue;			do {				t++;			} while (t[0] == '\x20' || t[0] == '\t'); 			begin = t;			while (isalnum(Py_CHARMASK(t[0])) ||			       t[0] == '-' || t[0] == '_' || t[0] == '.')				t++; 			if (begin < t) {				char* r = new_string(begin, t - begin);				char* q = get_normal_name(r);				if (r != q) {					PyMem_FREE(r);					r = new_string(q, strlen(q));				}				return r;			}		}	}	return NULL;} /* Check whether the line contains a coding spec. If it does,   invoke the set_readline function for the new encoding.   This function receives the tok_state and the new encoding.   Return 1 on success, 0 on failure.  */ static intcheck_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,		  int set_readline(struct tok_state *, const char *)){	char * cs;	int r = 1;         if (tok->cont_line)		/* It's a continuation line, so it can't be a coding spec. */		return 1;	cs = get_coding_spec(line, size);	if (cs != NULL) {		tok->read_coding_spec = 1;		if (tok->encoding == NULL) {			assert(tok->decoding_state == 1); /* raw */			if (strcmp(cs, "utf-8") == 0 ||			    strcmp(cs, "iso-8859-1") == 0) {				tok->encoding = cs;			} else {#ifdef Py_USING_UNICODE				r = set_readline(tok, cs);				if (r) {					tok->encoding = cs;					tok->decoding_state = -1;				}				else					PyMem_FREE(cs);#else                                /* Without Unicode support, we cannot                                   process the coding spec. Since there                                   won't be any Unicode literals, that                                   won't matter. */				PyMem_FREE(cs);#endif			}		} else {	/* then, compare cs with BOM */			r = (strcmp(tok->encoding, cs) == 0);			PyMem_FREE(cs);		}	}	if (!r) {		cs = tok->encoding;		if (!cs)			cs = "with BOM";		PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);	}	return r;} /* See whether the file starts with a BOM. If it does,   invoke the set_readline function with the new encoding.   Return 1 on success, 0 on failure.  */ static intcheck_bom(int get_char(struct tok_state *),	  void unget_char(int, struct tok_state *),	  int set_readline(struct tok_state *, const char *),	  struct tok_state *tok){	int ch = get_char(tok);	tok->decoding_state = 1;	if (ch == EOF) {		return 1;	} else if (ch == 0xEF) {		ch = get_char(tok); if (ch != 0xBB) goto NON_BOM;		ch = get_char(tok); if (ch != 0xBF) goto NON_BOM;#if 0	/* Disable support for UTF-16 BOMs until a decision	   is made whether this needs to be supported.  */	} else if (ch == 0xFE) {		ch = get_char(tok); if (ch != 0xFF) goto NON_BOM;		if (!set_readline(tok, "utf-16-be")) return 0;		tok->decoding_state = -1;	} else if (ch == 0xFF) {		ch = get_char(tok); if (ch != 0xFE) goto NON_BOM;		if (!set_readline(tok, "utf-16-le")) return 0;		tok->decoding_state = -1;#endif	} else {		unget_char(ch, tok);		return 1;	}	if (tok->encoding != NULL)		PyMem_FREE(tok->encoding);	tok->encoding = new_string("utf-8", 5);	/* resulting is in utf-8 */	return 1;  NON_BOM:	/* any token beginning with '\xEF', '\xFE', '\xFF' is a bad token */	unget_char(0xFF, tok);	/* XXX this will cause a syntax error */	return 1;} /* Read a line of text from TOK into S, using the stream in TOK.   Return NULL on failure, else S.    On entry, tok->decoding_buffer will be one of:     1) NULL: need to call tok->decoding_readline to get a new line     2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and           stored the result in tok->decoding_buffer     3) PyStringObject *: previous call to fp_readl did not have enough room           (in the s buffer) to copy entire contents of the line read           by tok->decoding_readline.  tok->decoding_buffer has the overflow.           In this case, fp_readl is called in a loop (with an expanded buffer)           until the buffer ends with a '\n' (or until the end of the file is           reached): see tok_nextc and its calls to decoding_fgets.*/ static char *fp_readl(char *s, int size, struct tok_state *tok){#ifndef Py_USING_UNICODE	/* In a non-Unicode built, this should never be called. */	Py_FatalError("fp_readl should not be called in this build.");	return NULL; /* Keep compiler happy (not reachable) */#else	PyObject* utf8 = NULL;	PyObject* buf = tok->decoding_buffer;	char *str;	Py_ssize_t utf8len; 	/* Ask for one less byte so we can terminate it */	assert(size > 0);	size--; 	if (buf == NULL) {		buf = PyObject_CallObject(tok->decoding_readline, NULL);		if (buf == NULL)			return error_ret(tok);	} else {		tok->decoding_buffer = NULL;		if (PyString_CheckExact(buf))			utf8 = buf;	}	if (utf8 == NULL) {		utf8 = PyUnicode_AsUTF8String(buf);		Py_DECREF(buf);		if (utf8 == NULL)			return error_ret(tok);	}	str = PyString_AsString(utf8);	utf8len = PyString_GET_SIZE(utf8);	if (utf8len > size) {		tok->decoding_buffer = PyString_FromStringAndSize(str+size, utf8len-size);		if (tok->decoding_buffer == NULL) {			Py_DECREF(utf8);			return error_ret(tok);		}		utf8len = size;	}	memcpy(s, str, utf8len);	s[utf8len] = '\0';	Py_DECREF(utf8);	if (utf8len == 0) return NULL; /* EOF */	return s;#endif} /* Set the readline function for TOK to a StreamReader's   readline function. The StreamReader is named ENC.    This function is called from check_bom and check_coding_spec.    ENC is usually identical to the future value of tok->encoding,   except for the (currently unsupported) case of UTF-16.    Return 1 on success, 0 on failure. */ static intfp_setreadl(struct tok_state *tok, const char* enc){	PyObject *reader, *stream, *readline; 	/* XXX: constify filename argument. */	stream = PyFile_FromFile(tok->fp, (char*)tok->filename, "rb", NULL);	if (stream == NULL)		return 0; 	reader = PyCodec_StreamReader(enc, stream, NULL);	Py_DECREF(stream);	if (reader == NULL)		return 0; 	readline = PyObject_GetAttrString(reader, "readline");	Py_DECREF(reader);	if (readline == NULL)		return 0; 	tok->decoding_readline = readline;	return 1;} /* Fetch the next byte from TOK. */ static int fp_getc(struct tok_state *tok) {	return getc(tok->fp);} /* Unfetch the last byte back into TOK.  */ static void fp_ungetc(int c, struct tok_state *tok) {	ungetc(c, tok->fp);} /* Read a line of input from TOK. Determine encoding   if necessary.  */ static char *decoding_fgets(char *s, int size, struct tok_state *tok){	char *line = NULL;	int badchar = 0;	for (;;) {		if (tok->decoding_state < 0) {			/* We already have a codec associated with			   this input. */			line = fp_readl(s, size, tok);			break;		} else if (tok->decoding_state > 0) {			/* We want a 'raw' read. */			line = Py_UniversalNewlineFgets(s, size,							tok->fp, NULL);			break;		} else {			/* We have not yet determined the encoding.			   If an encoding is found, use the file-pointer			   reader functions from now on. */			if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok))				return error_ret(tok);			assert(tok->decoding_state != 0);		}	}	if (line != NULL && tok->lineno < 2 && !tok->read_coding_spec) {		if (!check_coding_spec(line, strlen(line), tok, fp_setreadl)) {			return error_ret(tok);		}	}#ifndef PGEN	/* The default encoding is ASCII, so make sure we don't have any           non-ASCII bytes in it. */	if (line && !tok->encoding) {		unsigned char *c;		for (c = (unsigned char *)line; *c; c++)			if (*c > 127) {				badchar = *c;				break;			}

Lines 1–500 of 1535. Load more