*p++ = '-';                        inShift = 1;                    }                } else if (SPECIAL(ch,0,0)) {                    errmsg = "unexpected special character";	                goto utf7Error;                } else  {                    *p++ = ch;                }            } else {                charsleft = (charsleft << 6) | UB64(ch);                bitsleft += 6;                s++;                /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);            }        }        else if ( ch == '+' ) {            startinpos = s-starts;            s++;            if (s < e && *s == '-') {                s++;                *p++ = '+';            } else            {                inShift = 1;                bitsleft = 0;            }        }        else if (SPECIAL(ch,0,0)) {            errmsg = "unexpected special character";            s++;	        goto utf7Error;        }        else {            *p++ = ch;            s++;        }        continue;    utf7Error:        outpos = p-PyUnicode_AS_UNICODE(unicode);        endinpos = s-starts;        if (unicode_decode_call_errorhandler(             errors, &errorHandler,             "utf7", errmsg,             starts, size, &startinpos, &endinpos, &exc, &s,             (PyObject **)&unicode, &outpos, &p))        goto onError;    }     if (inShift) {        outpos = p-PyUnicode_AS_UNICODE(unicode);        endinpos = size;        if (unicode_decode_call_errorhandler(             errors, &errorHandler,             "utf7", "unterminated shift sequence",             starts, size, &startinpos, &endinpos, &exc, &s,             (PyObject **)&unicode, &outpos, &p))            goto onError;        if (s < e)           goto restart;    }     if (_PyUnicode_Resize(&unicode, p - PyUnicode_AS_UNICODE(unicode)) < 0)        goto onError;     Py_XDECREF(errorHandler);    Py_XDECREF(exc);    return (PyObject *)unicode; onError:    Py_XDECREF(errorHandler);    Py_XDECREF(exc);    Py_DECREF(unicode);    return NULL;}  PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,                   Py_ssize_t size,                   int encodeSetO,                   int encodeWhiteSpace,                   const char *errors){    PyObject *v;    /* It might be possible to tighten this worst case */    Py_ssize_t cbAllocated = 5 * size;    int inShift = 0;    Py_ssize_t i = 0;    unsigned int bitsleft = 0;    unsigned long charsleft = 0;    char * out;    char * start;     if (size == 0)		return PyString_FromStringAndSize(NULL, 0);     v = PyString_FromStringAndSize(NULL, cbAllocated);    if (v == NULL)        return NULL;     start = out = PyString_AS_STRING(v);    for (;i < size; ++i) {        Py_UNICODE ch = s[i];         if (!inShift) {            if (ch == '+') {                *out++ = '+';                *out++ = '-';            } else if (SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {                charsleft = ch;                bitsleft = 16;                *out++ = '+';                /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);                inShift = bitsleft > 0;            } else {                *out++ = (char) ch;            }        } else {            if (!SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {                *out++ = B64(charsleft << (6-bitsleft));                charsleft = 0;                bitsleft = 0;                /* Characters not in the BASE64 set implicitly unshift the sequence                   so no '-' is required, except if the character is itself a '-' */                if (B64CHAR(ch) || ch == '-') {                    *out++ = '-';                }                inShift = 0;                *out++ = (char) ch;            } else {                bitsleft += 16;                charsleft = (charsleft << 16) | ch;                /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);                 /* If the next character is special then we dont' need to terminate                   the shift sequence. If the next character is not a BASE64 character                   or '-' then the shift sequence will be terminated implicitly and we                   don't have to insert a '-'. */                 if (bitsleft == 0) {                    if (i + 1 < size) {                        Py_UNICODE ch2 = s[i+1];                         if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) {                         } else if (B64CHAR(ch2) || ch2 == '-') {                            *out++ = '-';                            inShift = 0;                        } else {                            inShift = 0;                        }                     }                    else {                        *out++ = '-';                        inShift = 0;                    }                }            }        }    }    if (bitsleft) {        *out++= B64(charsleft << (6-bitsleft) );        *out++ = '-';    }     _PyString_Resize(&v, out - start);    return v;} #undef SPECIAL#undef B64#undef B64CHAR#undef UB64#undef ENCODE#undef DECODE /* --- UTF-8 Codec -------------------------------------------------------- */ staticchar utf8_code_length[256] = {    /* Map UTF-8 encoded prefix byte to sequence length.  zero means       illegal prefix.  see RFC 2279 for details */    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,    2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,    3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,    4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0}; PyObject *PyUnicode_DecodeUTF8(const char *s,			       Py_ssize_t size,			       const char *errors){    return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);} PyObject *PyUnicode_DecodeUTF8Stateful(const char *s,			                Py_ssize_t size,			                const char *errors,			                Py_ssize_t *consumed){    const char *starts = s;    int n;    Py_ssize_t startinpos;    Py_ssize_t endinpos;    Py_ssize_t outpos;    const char *e;    PyUnicodeObject *unicode;    Py_UNICODE *p;    const char *errmsg = "";    PyObject *errorHandler = NULL;    PyObject *exc = NULL;     /* Note: size will always be longer than the resulting Unicode       character count */    unicode = _PyUnicode_New(size);    if (!unicode)        return NULL;    if (size == 0) {        if (consumed)            *consumed = 0;        return (PyObject *)unicode;    }     /* Unpack UTF-8 encoded data */    p = unicode->str;    e = s + size;     while (s < e) {        Py_UCS4 ch = (unsigned char)*s;         if (ch < 0x80) {            *p++ = (Py_UNICODE)ch;            s++;            continue;        }         n = utf8_code_length[ch];         if (s + n > e) {	    if (consumed)		break;	    else {		errmsg = "unexpected end of data";		startinpos = s-starts;		endinpos = size;		goto utf8Error;	    }	}         switch (n) {         case 0:            errmsg = "unexpected code byte";	    startinpos = s-starts;	    endinpos = startinpos+1;	    goto utf8Error;         case 1:            errmsg = "internal error";	    startinpos = s-starts;	    endinpos = startinpos+1;	    goto utf8Error;         case 2:            if ((s[1] & 0xc0) != 0x80) {                errmsg = "invalid data";		startinpos = s-starts;		endinpos = startinpos+2;		goto utf8Error;	    }            ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);            if (ch < 0x80) {		startinpos = s-starts;		endinpos = startinpos+2;                errmsg = "illegal encoding";		goto utf8Error;	    }	    else		*p++ = (Py_UNICODE)ch;            break;         case 3:            if ((s[1] & 0xc0) != 0x80 ||                (s[2] & 0xc0) != 0x80) {                errmsg = "invalid data";		startinpos = s-starts;		endinpos = startinpos+3;		goto utf8Error;	    }            ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);            if (ch < 0x0800) {		/* Note: UTF-8 encodings of surrogates are considered		   legal UTF-8 sequences; 		   XXX For wide builds (UCS-4) we should probably try		       to recombine the surrogates into a single code		       unit.		*/                errmsg = "illegal encoding";		startinpos = s-starts;		endinpos = startinpos+3;		goto utf8Error;	    }	    else		*p++ = (Py_UNICODE)ch;            break;         case 4:            if ((s[1] & 0xc0) != 0x80 ||                (s[2] & 0xc0) != 0x80 ||                (s[3] & 0xc0) != 0x80) {                errmsg = "invalid data";		startinpos = s-starts;		endinpos = startinpos+4;		goto utf8Error;	    }            ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +                 ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);            /* validate and convert to UTF-16 */            if ((ch < 0x10000)        /* minimum value allowed for 4					 byte encoding */                || (ch > 0x10ffff))   /* maximum value allowed for					 UTF-16 */	    {                errmsg = "illegal encoding";		startinpos = s-starts;		endinpos = startinpos+4;		goto utf8Error;	    }#ifdef Py_UNICODE_WIDE	    *p++ = (Py_UNICODE)ch;#else            /*  compute and append the two surrogates: */             /*  translate from 10000..10FFFF to 0..FFFF */            ch -= 0x10000;             /*  high surrogate = top 10 bits added to D800 */            *p++ = (Py_UNICODE)(0xD800 + (ch >> 10));             /*  low surrogate = bottom 10 bits added to DC00 */            *p++ = (Py_UNICODE)(0xDC00 + (ch & 0x03FF));#endif            break;         default:            /* Other sizes are only needed for UCS-4 */            errmsg = "unsupported Unicode code range";	    startinpos = s-starts;	    endinpos = startinpos+n;	    goto utf8Error;        }        s += n;	continue;     utf8Error:    outpos = p-PyUnicode_AS_UNICODE(unicode);    if (unicode_decode_call_errorhandler(	     errors, &errorHandler,	     "utf8", errmsg,	     starts, size, &startinpos, &endinpos, &exc, &s,	     (PyObject **)&unicode, &outpos, &p))	goto onError;    }    if (consumed)	*consumed = s-starts;     /* Adjust length */    if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0)        goto onError;     Py_XDECREF(errorHandler);    Py_XDECREF(exc);    return (PyObject *)unicode; onError:    Py_XDECREF(errorHandler);    Py_XDECREF(exc);    Py_DECREF(unicode);    return NULL;} /* Allocation strategy:  if the string is short, convert into a stack buffer   and allocate exactly as much space needed at the end.  Else allocate the   maximum possible needed (4 result bytes per Unicode character), and return   the excess memory at the end.*/PyObject *PyUnicode_EncodeUTF8(const Py_UNICODE *s,		     Py_ssize_t size,		     const char *errors){#define MAX_SHORT_UNICHARS 300  /* largest size we'll do on the stack */     Py_ssize_t i;           /* index into s of next input byte */    PyObject *v;        /* result string object */    char *p;            /* next free byte in output buffer */    Py_ssize_t nallocated;  /* number of result bytes allocated */    Py_ssize_t nneeded;        /* number of result bytes needed */    char stackbuf[MAX_SHORT_UNICHARS * 4];     assert(s != NULL);    assert(size >= 0);     if (size <= MAX_SHORT_UNICHARS) {        /* Write into the stack buffer; nallocated can't overflow.         * At the end, we'll allocate exactly as much heap space as it         * turns out we need.         */        nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);        v = NULL;   /* will allocate after we're done */        p = stackbuf;    }    else {        /* Overallocate on the heap, and give the excess back at the end. */        nallocated = size * 4;        if (nallocated / 4 != size)  /* overflow! */            return PyErr_NoMemory();        v = PyString_FromStringAndSize(NULL, nallocated);        if (v == NULL)            return NULL;        p = PyString_AS_STRING(v);    }     for (i = 0; i < size;) {        Py_UCS4 ch = s[i++];         if (ch < 0x80)            /* Encode ASCII */            *p++ = (char) ch;         else if (ch < 0x0800) {            /* Encode Latin-1 */            *p++ = (char)(0xc0 | (ch >> 6));            *p++ = (char)(0x80 | (ch & 0x3f));        }        else {            /* Encode UCS2 Unicode ordinals */            if (ch < 0x10000) {                /* Special case: check for high surrogate */                if (0xD800 <= ch && ch <= 0xDBFF && i != size) {                    Py_UCS4 ch2 = s[i];                    /* Check for low surrogate and combine the two to                       form a UCS4 value */                    if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {                        ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;                        i++;                        goto encodeUCS4;                    }                    /* Fall through: handles isolated high surrogates */                }                *p++ = (char)(0xe0 | (ch >> 12));                *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));                *p++ = (char)(0x80 | (ch & 0x3f));                continue;    	    }encodeUCS4:            /* Encode UCS4 Unicode ordinals */            *p++ = (char)(0xf0 | (ch >> 18));            *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));            *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));            *p++ = (char)(0x80 | (ch & 0x3f));        }    }     if (v == NULL) {        /* This was stack allocated. */        nneeded = p - stackbuf;        assert(nneeded <= nallocated);        v = PyString_FromStringAndSize(stackbuf, nneeded);    }    else {    	/* Cut back to size actually needed. */        nneeded = p - PyString_AS_STRING(v);        assert(nneeded <= nallocated);        _PyString_Resize(&v, nneeded);    }    return v; #undef MAX_SHORT_UNICHARS} PyObject *PyUnicode_AsUTF8String(PyObject *unicode){    if (!PyUnicode_Check(unicode)) {        PyErr_BadArgument();        return NULL;

Lines 1001–1500 of 8067. Previous lines Load more