History

add hg and python

cinap_lenrek committed 15 years ago

Commit 4bb9e221 parent 987265c3706 files · +1228772 −-233Browse files

Showing the changes to sys/src/cmd/python/Objects/unicodeobject.c (1 of 3706 files). Show all

Unified · Split

sys/src/cmd/python/Objects/unicodeobject.c+8067−0
@@ -0,0 +1,8067 @@
1+/*
2+
3+Unicode implementation based on original code by Fredrik Lundh,
4+modified by Marc-Andre Lemburg <mal@lemburg.com> according to the
5+Unicode Integration Proposal (see file Misc/unicode.txt).
6+
7+Major speed upgrades to the method implementations at the Reykjavik
8+NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke.
9+
10+Copyright (c) Corporation for National Research Initiatives.
11+
12+--------------------------------------------------------------------
13+The original string type implementation is:
14+
15+ Copyright (c) 1999 by Secret Labs AB
16+ Copyright (c) 1999 by Fredrik Lundh
17+
18+By obtaining, using, and/or copying this software and/or its
19+associated documentation, you agree that you have read, understood,
20+and will comply with the following terms and conditions:
21+
22+Permission to use, copy, modify, and distribute this software and its
23+associated documentation for any purpose and without fee is hereby
24+granted, provided that the above copyright notice appears in all
25+copies, and that both that copyright notice and this permission notice
26+appear in supporting documentation, and that the name of Secret Labs
27+AB or the author not be used in advertising or publicity pertaining to
28+distribution of the software without specific, written prior
29+permission.
30+
31+SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
32+THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
33+FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
34+ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
35+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
36+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
37+OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
38+--------------------------------------------------------------------
39+
40+*/
41+
42+#define PY_SSIZE_T_CLEAN
43+#include "Python.h"
44+
45+#include "unicodeobject.h"
46+#include "ucnhash.h"
47+
48+#ifdef MS_WINDOWS
49+#include <windows.h>
50+#endif
51+
52+/* Limit for the Unicode object free list */
53+
54+#define MAX_UNICODE_FREELIST_SIZE 1024
55+
56+/* Limit for the Unicode object free list stay alive optimization.
57+
58+ The implementation will keep allocated Unicode memory intact for
59+ all objects on the free list having a size less than this
60+ limit. This reduces malloc() overhead for small Unicode objects.
61+
62+ At worst this will result in MAX_UNICODE_FREELIST_SIZE *
63+ (sizeof(PyUnicodeObject) + KEEPALIVE_SIZE_LIMIT +
64+ malloc()-overhead) bytes of unused garbage.
65+
66+ Setting the limit to 0 effectively turns the feature off.
67+
68+ Note: This is an experimental feature ! If you get core dumps when
69+ using Unicode objects, turn this feature off.
70+
71+*/
72+
73+#define KEEPALIVE_SIZE_LIMIT 9
74+
75+/* Endianness switches; defaults to little endian */
76+
77+#ifdef WORDS_BIGENDIAN
78+# define BYTEORDER_IS_BIG_ENDIAN
79+#else
80+# define BYTEORDER_IS_LITTLE_ENDIAN
81+#endif
82+
83+/* --- Globals ------------------------------------------------------------
84+
85+ The globals are initialized by the _PyUnicode_Init() API and should
86+ not be used before calling that API.
87+
88+*/
89+
90+
91+#ifdef __cplusplus
92+extern "C" {
93+#endif
94+
95+/* Free list for Unicode objects */
96+static PyUnicodeObject *unicode_freelist;
97+static int unicode_freelist_size;
98+
99+/* The empty Unicode object is shared to improve performance. */
100+static PyUnicodeObject *unicode_empty;
101+
102+/* Single character Unicode strings in the Latin-1 range are being
103+ shared as well. */
104+static PyUnicodeObject *unicode_latin1[256];
105+
106+/* Default encoding to use and assume when NULL is passed as encoding
107+ parameter; it is initialized by _PyUnicode_Init().
108+
109+ Always use the PyUnicode_SetDefaultEncoding() and
110+ PyUnicode_GetDefaultEncoding() APIs to access this global.
111+
112+*/
113+static char unicode_default_encoding[100];
114+
115+Py_UNICODE
116+PyUnicode_GetMax(void)
117+{
118+#ifdef Py_UNICODE_WIDE
119+ return 0x10FFFF;
120+#else
121+ /* This is actually an illegal character, so it should
122+ not be passed to unichr. */
123+ return 0xFFFF;
124+#endif
125+}
126+
127+/* --- Bloom Filters ----------------------------------------------------- */
128+
129+/* stuff to implement simple "bloom filters" for Unicode characters.
130+ to keep things simple, we use a single bitmask, using the least 5
131+ bits from each unicode characters as the bit index. */
132+
133+/* the linebreak mask is set up by Unicode_Init below */
134+
135+#define BLOOM_MASK unsigned long
136+
137+static BLOOM_MASK bloom_linebreak;
138+
139+#define BLOOM(mask, ch) ((mask & (1 << ((ch) & 0x1F))))
140+
141+#define BLOOM_LINEBREAK(ch)\
142+ (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK((ch)))
143+
144+Py_LOCAL_INLINE(BLOOM_MASK) make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len)
145+{
146+ /* calculate simple bloom-style bitmask for a given unicode string */
147+
148+ long mask;
149+ Py_ssize_t i;
150+
151+ mask = 0;
152+ for (i = 0; i < len; i++)
153+ mask |= (1 << (ptr[i] & 0x1F));
154+
155+ return mask;
156+}
157+
158+Py_LOCAL_INLINE(int) unicode_member(Py_UNICODE chr, Py_UNICODE* set, Py_ssize_t setlen)
159+{
160+ Py_ssize_t i;
161+
162+ for (i = 0; i < setlen; i++)
163+ if (set[i] == chr)
164+ return 1;
165+
166+ return 0;
167+}
168+
169+#define BLOOM_MEMBER(mask, chr, set, setlen)\
170+ BLOOM(mask, chr) && unicode_member(chr, set, setlen)
171+
172+/* --- Unicode Object ----------------------------------------------------- */
173+
174+static
175+int unicode_resize(register PyUnicodeObject *unicode,
176+ Py_ssize_t length)
177+{
178+ void *oldstr;
179+
180+ /* Shortcut if there's nothing much to do. */
181+ if (unicode->length == length)
182+ goto reset;
183+
184+ /* Resizing shared object (unicode_empty or single character
185+ objects) in-place is not allowed. Use PyUnicode_Resize()
186+ instead ! */
187+
188+ if (unicode == unicode_empty ||
189+ (unicode->length == 1 &&
190+ unicode->str[0] < 256U &&
191+ unicode_latin1[unicode->str[0]] == unicode)) {
192+ PyErr_SetString(PyExc_SystemError,
193+ "can't resize shared unicode objects");
194+ return -1;
195+ }
196+
197+ /* We allocate one more byte to make sure the string is Ux0000 terminated.
198+ The overallocation is also used by fastsearch, which assumes that it's
199+ safe to look at str[length] (without making any assumptions about what
200+ it contains). */
201+
202+ oldstr = unicode->str;
203+ PyMem_RESIZE(unicode->str, Py_UNICODE, length + 1);
204+ if (!unicode->str) {
205+ unicode->str = (Py_UNICODE *)oldstr;
206+ PyErr_NoMemory();
207+ return -1;
208+ }
209+ unicode->str[length] = 0;
210+ unicode->length = length;
211+
212+ reset:
213+ /* Reset the object caches */
214+ if (unicode->defenc) {
215+ Py_DECREF(unicode->defenc);
216+ unicode->defenc = NULL;
217+ }
218+ unicode->hash = -1;
219+
220+ return 0;
221+}
222+
223+/* We allocate one more byte to make sure the string is
224+ Ux0000 terminated -- XXX is this needed ?
225+
226+ XXX This allocator could further be enhanced by assuring that the
227+ free list never reduces its size below 1.
228+
229+*/
230+
231+static
232+PyUnicodeObject *_PyUnicode_New(Py_ssize_t length)
233+{
234+ register PyUnicodeObject *unicode;
235+
236+ /* Optimization for empty strings */
237+ if (length == 0 && unicode_empty != NULL) {
238+ Py_INCREF(unicode_empty);
239+ return unicode_empty;
240+ }
241+
242+ /* Unicode freelist & memory allocation */
243+ if (unicode_freelist) {
244+ unicode = unicode_freelist;
245+ unicode_freelist = *(PyUnicodeObject **)unicode;
246+ unicode_freelist_size--;
247+ if (unicode->str) {
248+ /* Keep-Alive optimization: we only upsize the buffer,
249+ never downsize it. */
250+ if ((unicode->length < length) &&
251+ unicode_resize(unicode, length) < 0) {
252+ PyMem_DEL(unicode->str);
253+ goto onError;
254+ }
255+ }
256+ else {
257+ unicode->str = PyMem_NEW(Py_UNICODE, length + 1);
258+ }
259+ PyObject_INIT(unicode, &PyUnicode_Type);
260+ }
261+ else {
262+ unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);
263+ if (unicode == NULL)
264+ return NULL;
265+ unicode->str = PyMem_NEW(Py_UNICODE, length + 1);
266+ }
267+
268+ if (!unicode->str) {
269+ PyErr_NoMemory();
270+ goto onError;
271+ }
272+ /* Initialize the first element to guard against cases where
273+ * the caller fails before initializing str -- unicode_resize()
274+ * reads str[0], and the Keep-Alive optimization can keep memory
275+ * allocated for str alive across a call to unicode_dealloc(unicode).
276+ * We don't want unicode_resize to read uninitialized memory in
277+ * that case.
278+ */
279+ unicode->str[0] = 0;
280+ unicode->str[length] = 0;
281+ unicode->length = length;
282+ unicode->hash = -1;
283+ unicode->defenc = NULL;
284+ return unicode;
285+
286+ onError:
287+ _Py_ForgetReference((PyObject *)unicode);
288+ PyObject_Del(unicode);
289+ return NULL;
290+}
291+
292+static
293+void unicode_dealloc(register PyUnicodeObject *unicode)
294+{
295+ if (PyUnicode_CheckExact(unicode) &&
296+ unicode_freelist_size < MAX_UNICODE_FREELIST_SIZE) {
297+ /* Keep-Alive optimization */
298+ if (unicode->length >= KEEPALIVE_SIZE_LIMIT) {
299+ PyMem_DEL(unicode->str);
300+ unicode->str = NULL;
301+ unicode->length = 0;
302+ }
303+ if (unicode->defenc) {
304+ Py_DECREF(unicode->defenc);
305+ unicode->defenc = NULL;
306+ }
307+ /* Add to free list */
308+ *(PyUnicodeObject **)unicode = unicode_freelist;
309+ unicode_freelist = unicode;
310+ unicode_freelist_size++;
311+ }
312+ else {
313+ PyMem_DEL(unicode->str);
314+ Py_XDECREF(unicode->defenc);
315+ unicode->ob_type->tp_free((PyObject *)unicode);
316+ }
317+}
318+
319+int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length)
320+{
321+ register PyUnicodeObject *v;
322+
323+ /* Argument checks */
324+ if (unicode == NULL) {
325+ PyErr_BadInternalCall();
326+ return -1;
327+ }
328+ v = (PyUnicodeObject *)*unicode;
329+ if (v == NULL || !PyUnicode_Check(v) || v->ob_refcnt != 1 || length < 0) {
330+ PyErr_BadInternalCall();
331+ return -1;
332+ }
333+
334+ /* Resizing unicode_empty and single character objects is not
335+ possible since these are being shared. We simply return a fresh
336+ copy with the same Unicode content. */
337+ if (v->length != length &&
338+ (v == unicode_empty || v->length == 1)) {
339+ PyUnicodeObject *w = _PyUnicode_New(length);
340+ if (w == NULL)
341+ return -1;
342+ Py_UNICODE_COPY(w->str, v->str,
343+ length < v->length ? length : v->length);
344+ Py_DECREF(*unicode);
345+ *unicode = (PyObject *)w;
346+ return 0;
347+ }
348+
349+ /* Note that we don't have to modify *unicode for unshared Unicode
350+ objects, since we can modify them in-place. */
351+ return unicode_resize(v, length);
352+}
353+
354+/* Internal API for use in unicodeobject.c only ! */
355+#define _PyUnicode_Resize(unicodevar, length) \
356+ PyUnicode_Resize(((PyObject **)(unicodevar)), length)
357+
358+PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u,
359+ Py_ssize_t size)
360+{
361+ PyUnicodeObject *unicode;
362+
363+ /* If the Unicode data is known at construction time, we can apply
364+ some optimizations which share commonly used objects. */
365+ if (u != NULL) {
366+
367+ /* Optimization for empty strings */
368+ if (size == 0 && unicode_empty != NULL) {
369+ Py_INCREF(unicode_empty);
370+ return (PyObject *)unicode_empty;
371+ }
372+
373+ /* Single character Unicode objects in the Latin-1 range are
374+ shared when using this constructor */
375+ if (size == 1 && *u < 256) {
376+ unicode = unicode_latin1[*u];
377+ if (!unicode) {
378+ unicode = _PyUnicode_New(1);
379+ if (!unicode)
380+ return NULL;
381+ unicode->str[0] = *u;
382+ unicode_latin1[*u] = unicode;
383+ }
384+ Py_INCREF(unicode);
385+ return (PyObject *)unicode;
386+ }
387+ }
388+
389+ unicode = _PyUnicode_New(size);
390+ if (!unicode)
391+ return NULL;
392+
393+ /* Copy the Unicode data into the new object */
394+ if (u != NULL)
395+ Py_UNICODE_COPY(unicode->str, u, size);
396+
397+ return (PyObject *)unicode;
398+}
399+
400+#ifdef HAVE_WCHAR_H
401+
402+PyObject *PyUnicode_FromWideChar(register const wchar_t *w,
403+ Py_ssize_t size)
404+{
405+ PyUnicodeObject *unicode;
406+
407+ if (w == NULL) {
408+ PyErr_BadInternalCall();
409+ return NULL;
410+ }
411+
412+ unicode = _PyUnicode_New(size);
413+ if (!unicode)
414+ return NULL;
415+
416+ /* Copy the wchar_t data into the new object */
417+#ifdef HAVE_USABLE_WCHAR_T
418+ memcpy(unicode->str, w, size * sizeof(wchar_t));
419+#else
420+ {
421+ register Py_UNICODE *u;
422+ register Py_ssize_t i;
423+ u = PyUnicode_AS_UNICODE(unicode);
424+ for (i = size; i > 0; i--)
425+ *u++ = *w++;
426+ }
427+#endif
428+
429+ return (PyObject *)unicode;
430+}
431+
432+Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *unicode,
433+ wchar_t *w,
434+ Py_ssize_t size)
435+{
436+ if (unicode == NULL) {
437+ PyErr_BadInternalCall();
438+ return -1;
439+ }
440+
441+ /* If possible, try to copy the 0-termination as well */
442+ if (size > PyUnicode_GET_SIZE(unicode))
443+ size = PyUnicode_GET_SIZE(unicode) + 1;
444+
445+#ifdef HAVE_USABLE_WCHAR_T
446+ memcpy(w, unicode->str, size * sizeof(wchar_t));
447+#else
448+ {
449+ register Py_UNICODE *u;
450+ register Py_ssize_t i;
451+ u = PyUnicode_AS_UNICODE(unicode);
452+ for (i = size; i > 0; i--)
453+ *w++ = *u++;
454+ }
455+#endif
456+
457+ if (size > PyUnicode_GET_SIZE(unicode))
458+ return PyUnicode_GET_SIZE(unicode);
459+ else
460+ return size;
461+}
462+
463+#endif
464+
465+PyObject *PyUnicode_FromOrdinal(int ordinal)
466+{
467+ Py_UNICODE s[1];
468+
469+#ifdef Py_UNICODE_WIDE
470+ if (ordinal < 0 || ordinal > 0x10ffff) {
471+ PyErr_SetString(PyExc_ValueError,
472+ "unichr() arg not in range(0x110000) "
473+ "(wide Python build)");
474+ return NULL;
475+ }
476+#else
477+ if (ordinal < 0 || ordinal > 0xffff) {
478+ PyErr_SetString(PyExc_ValueError,
479+ "unichr() arg not in range(0x10000) "
480+ "(narrow Python build)");
481+ return NULL;
482+ }
483+#endif
484+
485+ s[0] = (Py_UNICODE)ordinal;
486+ return PyUnicode_FromUnicode(s, 1);
487+}
488+
489+PyObject *PyUnicode_FromObject(register PyObject *obj)
490+{
491+ /* XXX Perhaps we should make this API an alias of
492+ PyObject_Unicode() instead ?! */
493+ if (PyUnicode_CheckExact(obj)) {
494+ Py_INCREF(obj);
495+ return obj;
496+ }
497+ if (PyUnicode_Check(obj)) {
498+ /* For a Unicode subtype that's not a Unicode object,
499+ return a true Unicode object with the same data. */
500+ return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj),
501+ PyUnicode_GET_SIZE(obj));
502+ }
503+ return PyUnicode_FromEncodedObject(obj, NULL, "strict");
504+}
505+
506+PyObject *PyUnicode_FromEncodedObject(register PyObject *obj,
507+ const char *encoding,
508+ const char *errors)
509+{
510+ const char *s = NULL;
511+ Py_ssize_t len;
512+ PyObject *v;
513+
514+ if (obj == NULL) {
515+ PyErr_BadInternalCall();
516+ return NULL;
517+ }
518+
519+#if 0
520+ /* For b/w compatibility we also accept Unicode objects provided
521+ that no encodings is given and then redirect to
522+ PyObject_Unicode() which then applies the additional logic for
523+ Unicode subclasses.
524+
525+ NOTE: This API should really only be used for object which
526+ represent *encoded* Unicode !
527+
528+ */
529+ if (PyUnicode_Check(obj)) {
530+ if (encoding) {
531+ PyErr_SetString(PyExc_TypeError,
532+ "decoding Unicode is not supported");
533+ return NULL;
534+ }
535+ return PyObject_Unicode(obj);
536+ }
537+#else
538+ if (PyUnicode_Check(obj)) {
539+ PyErr_SetString(PyExc_TypeError,
540+ "decoding Unicode is not supported");
541+ return NULL;
542+ }
543+#endif
544+
545+ /* Coerce object */
546+ if (PyString_Check(obj)) {
547+ s = PyString_AS_STRING(obj);
548+ len = PyString_GET_SIZE(obj);
549+ }
550+ else if (PyObject_AsCharBuffer(obj, &s, &len)) {
551+ /* Overwrite the error message with something more useful in
552+ case of a TypeError. */
553+ if (PyErr_ExceptionMatches(PyExc_TypeError))
554+ PyErr_Format(PyExc_TypeError,
555+ "coercing to Unicode: need string or buffer, "
556+ "%.80s found",
557+ obj->ob_type->tp_name);
558+ goto onError;
559+ }
560+
561+ /* Convert to Unicode */
562+ if (len == 0) {
563+ Py_INCREF(unicode_empty);
564+ v = (PyObject *)unicode_empty;
565+ }
566+ else
567+ v = PyUnicode_Decode(s, len, encoding, errors);
568+
569+ return v;
570+
571+ onError:
572+ return NULL;
573+}
574+
575+PyObject *PyUnicode_Decode(const char *s,
576+ Py_ssize_t size,
577+ const char *encoding,
578+ const char *errors)
579+{
580+ PyObject *buffer = NULL, *unicode;
581+
582+ if (encoding == NULL)
583+ encoding = PyUnicode_GetDefaultEncoding();
584+
585+ /* Shortcuts for common default encodings */
586+ if (strcmp(encoding, "utf-8") == 0)
587+ return PyUnicode_DecodeUTF8(s, size, errors);
588+ else if (strcmp(encoding, "latin-1") == 0)
589+ return PyUnicode_DecodeLatin1(s, size, errors);
590+#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
591+ else if (strcmp(encoding, "mbcs") == 0)
592+ return PyUnicode_DecodeMBCS(s, size, errors);
593+#endif
594+ else if (strcmp(encoding, "ascii") == 0)
595+ return PyUnicode_DecodeASCII(s, size, errors);
596+
597+ /* Decode via the codec registry */
598+ buffer = PyBuffer_FromMemory((void *)s, size);
599+ if (buffer == NULL)
600+ goto onError;
601+ unicode = PyCodec_Decode(buffer, encoding, errors);
602+ if (unicode == NULL)
603+ goto onError;
604+ if (!PyUnicode_Check(unicode)) {
605+ PyErr_Format(PyExc_TypeError,
606+ "decoder did not return an unicode object (type=%.400s)",
607+ unicode->ob_type->tp_name);
608+ Py_DECREF(unicode);
609+ goto onError;
610+ }
611+ Py_DECREF(buffer);
612+ return unicode;
613+
614+ onError:
615+ Py_XDECREF(buffer);
616+ return NULL;
617+}
618+
619+PyObject *PyUnicode_AsDecodedObject(PyObject *unicode,
620+ const char *encoding,
621+ const char *errors)
622+{
623+ PyObject *v;
624+
625+ if (!PyUnicode_Check(unicode)) {
626+ PyErr_BadArgument();
627+ goto onError;
628+ }
629+
630+ if (encoding == NULL)
631+ encoding = PyUnicode_GetDefaultEncoding();
632+
633+ /* Decode via the codec registry */
634+ v = PyCodec_Decode(unicode, encoding, errors);
635+ if (v == NULL)
636+ goto onError;
637+ return v;
638+
639+ onError:
640+ return NULL;
641+}
642+
643+PyObject *PyUnicode_Encode(const Py_UNICODE *s,
644+ Py_ssize_t size,
645+ const char *encoding,
646+ const char *errors)
647+{
648+ PyObject *v, *unicode;
649+
650+ unicode = PyUnicode_FromUnicode(s, size);
651+ if (unicode == NULL)
652+ return NULL;
653+ v = PyUnicode_AsEncodedString(unicode, encoding, errors);
654+ Py_DECREF(unicode);
655+ return v;
656+}
657+
658+PyObject *PyUnicode_AsEncodedObject(PyObject *unicode,
659+ const char *encoding,
660+ const char *errors)
661+{
662+ PyObject *v;
663+
664+ if (!PyUnicode_Check(unicode)) {
665+ PyErr_BadArgument();
666+ goto onError;
667+ }
668+
669+ if (encoding == NULL)
670+ encoding = PyUnicode_GetDefaultEncoding();
671+
672+ /* Encode via the codec registry */
673+ v = PyCodec_Encode(unicode, encoding, errors);
674+ if (v == NULL)
675+ goto onError;
676+ return v;
677+
678+ onError:
679+ return NULL;
680+}
681+
682+PyObject *PyUnicode_AsEncodedString(PyObject *unicode,
683+ const char *encoding,
684+ const char *errors)
685+{
686+ PyObject *v;
687+
688+ if (!PyUnicode_Check(unicode)) {
689+ PyErr_BadArgument();
690+ goto onError;
691+ }
692+
693+ if (encoding == NULL)
694+ encoding = PyUnicode_GetDefaultEncoding();
695+
696+ /* Shortcuts for common default encodings */
697+ if (errors == NULL) {
698+ if (strcmp(encoding, "utf-8") == 0)
699+ return PyUnicode_AsUTF8String(unicode);
700+ else if (strcmp(encoding, "latin-1") == 0)
701+ return PyUnicode_AsLatin1String(unicode);
702+#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
703+ else if (strcmp(encoding, "mbcs") == 0)
704+ return PyUnicode_AsMBCSString(unicode);
705+#endif
706+ else if (strcmp(encoding, "ascii") == 0)
707+ return PyUnicode_AsASCIIString(unicode);
708+ }
709+
710+ /* Encode via the codec registry */
711+ v = PyCodec_Encode(unicode, encoding, errors);
712+ if (v == NULL)
713+ goto onError;
714+ if (!PyString_Check(v)) {
715+ PyErr_Format(PyExc_TypeError,
716+ "encoder did not return a string object (type=%.400s)",
717+ v->ob_type->tp_name);
718+ Py_DECREF(v);
719+ goto onError;
720+ }
721+ return v;
722+
723+ onError:
724+ return NULL;
725+}
726+
727+PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,
728+ const char *errors)
729+{
730+ PyObject *v = ((PyUnicodeObject *)unicode)->defenc;
731+
732+ if (v)
733+ return v;
734+ v = PyUnicode_AsEncodedString(unicode, NULL, errors);
735+ if (v && errors == NULL)
736+ ((PyUnicodeObject *)unicode)->defenc = v;
737+ return v;
738+}
739+
740+Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode)
741+{
742+ if (!PyUnicode_Check(unicode)) {
743+ PyErr_BadArgument();
744+ goto onError;
745+ }
746+ return PyUnicode_AS_UNICODE(unicode);
747+
748+ onError:
749+ return NULL;
750+}
751+
752+Py_ssize_t PyUnicode_GetSize(PyObject *unicode)
753+{
754+ if (!PyUnicode_Check(unicode)) {
755+ PyErr_BadArgument();
756+ goto onError;
757+ }
758+ return PyUnicode_GET_SIZE(unicode);
759+
760+ onError:
761+ return -1;
762+}
763+
764+const char *PyUnicode_GetDefaultEncoding(void)
765+{
766+ return unicode_default_encoding;
767+}
768+
769+int PyUnicode_SetDefaultEncoding(const char *encoding)
770+{
771+ PyObject *v;
772+
773+ /* Make sure the encoding is valid. As side effect, this also
774+ loads the encoding into the codec registry cache. */
775+ v = _PyCodec_Lookup(encoding);
776+ if (v == NULL)
777+ goto onError;
778+ Py_DECREF(v);
779+ strncpy(unicode_default_encoding,
780+ encoding,
781+ sizeof(unicode_default_encoding));
782+ return 0;
783+
784+ onError:
785+ return -1;
786+}
787+
788+/* error handling callback helper:
789+ build arguments, call the callback and check the arguments,
790+ if no exception occurred, copy the replacement to the output
791+ and adjust various state variables.
792+ return 0 on success, -1 on error
793+*/
794+
795+static
796+int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
797+ const char *encoding, const char *reason,
798+ const char *input, Py_ssize_t insize, Py_ssize_t *startinpos, Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
799+ PyObject **output, Py_ssize_t *outpos, Py_UNICODE **outptr)
800+{
801+ static char *argparse = "O!n;decoding error handler must return (unicode, int) tuple";
802+
803+ PyObject *restuple = NULL;
804+ PyObject *repunicode = NULL;
805+ Py_ssize_t outsize = PyUnicode_GET_SIZE(*output);
806+ Py_ssize_t requiredsize;
807+ Py_ssize_t newpos;
808+ Py_UNICODE *repptr;
809+ Py_ssize_t repsize;
810+ int res = -1;
811+
812+ if (*errorHandler == NULL) {
813+ *errorHandler = PyCodec_LookupError(errors);
814+ if (*errorHandler == NULL)
815+ goto onError;
816+ }
817+
818+ if (*exceptionObject == NULL) {
819+ *exceptionObject = PyUnicodeDecodeError_Create(
820+ encoding, input, insize, *startinpos, *endinpos, reason);
821+ if (*exceptionObject == NULL)
822+ goto onError;
823+ }
824+ else {
825+ if (PyUnicodeDecodeError_SetStart(*exceptionObject, *startinpos))
826+ goto onError;
827+ if (PyUnicodeDecodeError_SetEnd(*exceptionObject, *endinpos))
828+ goto onError;
829+ if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason))
830+ goto onError;
831+ }
832+
833+ restuple = PyObject_CallFunctionObjArgs(*errorHandler, *exceptionObject, NULL);
834+ if (restuple == NULL)
835+ goto onError;
836+ if (!PyTuple_Check(restuple)) {
837+ PyErr_Format(PyExc_TypeError, &argparse[4]);
838+ goto onError;
839+ }
840+ if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos))
841+ goto onError;
842+ if (newpos<0)
843+ newpos = insize+newpos;
844+ if (newpos<0 || newpos>insize) {
845+ PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", newpos);
846+ goto onError;
847+ }
848+
849+ /* need more space? (at least enough for what we
850+ have+the replacement+the rest of the string (starting
851+ at the new input position), so we won't have to check space
852+ when there are no errors in the rest of the string) */
853+ repptr = PyUnicode_AS_UNICODE(repunicode);
854+ repsize = PyUnicode_GET_SIZE(repunicode);
855+ requiredsize = *outpos + repsize + insize-newpos;
856+ if (requiredsize > outsize) {
857+ if (requiredsize<2*outsize)
858+ requiredsize = 2*outsize;
859+ if (PyUnicode_Resize(output, requiredsize) < 0)
860+ goto onError;
861+ *outptr = PyUnicode_AS_UNICODE(*output) + *outpos;
862+ }
863+ *endinpos = newpos;
864+ *inptr = input + newpos;
865+ Py_UNICODE_COPY(*outptr, repptr, repsize);
866+ *outptr += repsize;
867+ *outpos += repsize;
868+ /* we made it! */
869+ res = 0;
870+
871+ onError:
872+ Py_XDECREF(restuple);
873+ return res;
874+}
875+
876+/* --- UTF-7 Codec -------------------------------------------------------- */
877+
878+/* see RFC2152 for details */
879+
880+static
881+char utf7_special[128] = {
882+ /* indicate whether a UTF-7 character is special i.e. cannot be directly
883+ encoded:
884+ 0 - not special
885+ 1 - special
886+ 2 - whitespace (optional)
887+ 3 - RFC2152 Set O (optional) */
888+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,
889+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
890+ 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,
891+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,
892+ 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
893+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,
894+ 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
895+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,
896+
897+};
898+
899+/* Note: The comparison (c) <= 0 is a trick to work-around gcc
900+ warnings about the comparison always being false; since
901+ utf7_special[0] is 1, we can safely make that one comparison
902+ true */
903+
904+#define SPECIAL(c, encodeO, encodeWS) \
905+ ((c) > 127 || (c) <= 0 || utf7_special[(c)] == 1 || \
906+ (encodeWS && (utf7_special[(c)] == 2)) || \
907+ (encodeO && (utf7_special[(c)] == 3)))
908+
909+#define B64(n) \
910+ ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f])
911+#define B64CHAR(c) \
912+ (isalnum(c) || (c) == '+' || (c) == '/')
913+#define UB64(c) \
914+ ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? \
915+ (c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4 )
916+
917+#define ENCODE(out, ch, bits) \
918+ while (bits >= 6) { \
919+ *out++ = B64(ch >> (bits-6)); \
920+ bits -= 6; \
921+ }
922+
923+#define DECODE(out, ch, bits, surrogate) \
924+ while (bits >= 16) { \
925+ Py_UNICODE outCh = (Py_UNICODE) ((ch >> (bits-16)) & 0xffff); \
926+ bits -= 16; \
927+ if (surrogate) { \
928+ /* We have already generated an error for the high surrogate \
929+ so let's not bother seeing if the low surrogate is correct or not */ \
930+ surrogate = 0; \
931+ } else if (0xDC00 <= outCh && outCh <= 0xDFFF) { \
932+ /* This is a surrogate pair. Unfortunately we can't represent \
933+ it in a 16-bit character */ \
934+ surrogate = 1; \
935+ errmsg = "code pairs are not supported"; \
936+ goto utf7Error; \
937+ } else { \
938+ *out++ = outCh; \
939+ } \
940+ }
941+
942+PyObject *PyUnicode_DecodeUTF7(const char *s,
943+ Py_ssize_t size,
944+ const char *errors)
945+{
946+ const char *starts = s;
947+ Py_ssize_t startinpos;
948+ Py_ssize_t endinpos;
949+ Py_ssize_t outpos;
950+ const char *e;
951+ PyUnicodeObject *unicode;
952+ Py_UNICODE *p;
953+ const char *errmsg = "";
954+ int inShift = 0;
955+ unsigned int bitsleft = 0;
956+ unsigned long charsleft = 0;
957+ int surrogate = 0;
958+ PyObject *errorHandler = NULL;
959+ PyObject *exc = NULL;
960+
961+ unicode = _PyUnicode_New(size);
962+ if (!unicode)
963+ return NULL;
964+ if (size == 0)
965+ return (PyObject *)unicode;
966+
967+ p = unicode->str;
968+ e = s + size;
969+
970+ while (s < e) {
971+ Py_UNICODE ch;
972+ restart:
973+ ch = *s;
974+
975+ if (inShift) {
976+ if ((ch == '-') || !B64CHAR(ch)) {
977+ inShift = 0;
978+ s++;
979+
980+ /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
981+ if (bitsleft >= 6) {
982+ /* The shift sequence has a partial character in it. If
983+ bitsleft < 6 then we could just classify it as padding
984+ but that is not the case here */
985+
986+ errmsg = "partial character in shift sequence";
987+ goto utf7Error;
988+ }
989+ /* According to RFC2152 the remaining bits should be zero. We
990+ choose to signal an error/insert a replacement character
991+ here so indicate the potential of a misencoded character. */
992+
993+ /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */
994+ if (bitsleft && charsleft << (sizeof(charsleft) * 8 - bitsleft)) {
995+ errmsg = "non-zero padding bits in shift sequence";
996+ goto utf7Error;
997+ }
998+
999+ if (ch == '-') {
1000+ if ((s < e) && (*(s) == '-')) {
1001+ *p++ = '-';
1002+ inShift = 1;
1003+ }
1004+ } else if (SPECIAL(ch,0,0)) {
1005+ errmsg = "unexpected special character";
1006+ goto utf7Error;
1007+ } else {
1008+ *p++ = ch;
1009+ }
1010+ } else {
1011+ charsleft = (charsleft << 6) | UB64(ch);
1012+ bitsleft += 6;
1013+ s++;
1014+ /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
1015+ }
1016+ }
1017+ else if ( ch == '+' ) {
1018+ startinpos = s-starts;
1019+ s++;
1020+ if (s < e && *s == '-') {
1021+ s++;
1022+ *p++ = '+';
1023+ } else
1024+ {
1025+ inShift = 1;
1026+ bitsleft = 0;
1027+ }
1028+ }
1029+ else if (SPECIAL(ch,0,0)) {
1030+ errmsg = "unexpected special character";
1031+ s++;
1032+ goto utf7Error;
1033+ }
1034+ else {
1035+ *p++ = ch;
1036+ s++;
1037+ }
1038+ continue;
1039+ utf7Error:
1040+ outpos = p-PyUnicode_AS_UNICODE(unicode);
1041+ endinpos = s-starts;
1042+ if (unicode_decode_call_errorhandler(
1043+ errors, &errorHandler,
1044+ "utf7", errmsg,
1045+ starts, size, &startinpos, &endinpos, &exc, &s,
1046+ (PyObject **)&unicode, &outpos, &p))
1047+ goto onError;
1048+ }
1049+
1050+ if (inShift) {
1051+ outpos = p-PyUnicode_AS_UNICODE(unicode);
1052+ endinpos = size;
1053+ if (unicode_decode_call_errorhandler(
1054+ errors, &errorHandler,
1055+ "utf7", "unterminated shift sequence",
1056+ starts, size, &startinpos, &endinpos, &exc, &s,
1057+ (PyObject **)&unicode, &outpos, &p))
1058+ goto onError;
1059+ if (s < e)
1060+ goto restart;
1061+ }
1062+
1063+ if (_PyUnicode_Resize(&unicode, p - PyUnicode_AS_UNICODE(unicode)) < 0)
1064+ goto onError;
1065+
1066+ Py_XDECREF(errorHandler);
1067+ Py_XDECREF(exc);
1068+ return (PyObject *)unicode;
1069+
1070+onError:
1071+ Py_XDECREF(errorHandler);
1072+ Py_XDECREF(exc);
1073+ Py_DECREF(unicode);
1074+ return NULL;
1075+}
1076+
1077+
1078+PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
1079+ Py_ssize_t size,
1080+ int encodeSetO,
1081+ int encodeWhiteSpace,
1082+ const char *errors)
1083+{
1084+ PyObject *v;
1085+ /* It might be possible to tighten this worst case */
1086+ Py_ssize_t cbAllocated = 5 * size;
1087+ int inShift = 0;
1088+ Py_ssize_t i = 0;
1089+ unsigned int bitsleft = 0;
1090+ unsigned long charsleft = 0;
1091+ char * out;
1092+ char * start;
1093+
1094+ if (size == 0)
1095+ return PyString_FromStringAndSize(NULL, 0);
1096+
1097+ v = PyString_FromStringAndSize(NULL, cbAllocated);
1098+ if (v == NULL)
1099+ return NULL;
1100+
1101+ start = out = PyString_AS_STRING(v);
1102+ for (;i < size; ++i) {
1103+ Py_UNICODE ch = s[i];
1104+
1105+ if (!inShift) {
1106+ if (ch == '+') {
1107+ *out++ = '+';
1108+ *out++ = '-';
1109+ } else if (SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
1110+ charsleft = ch;
1111+ bitsleft = 16;
1112+ *out++ = '+';
1113+ /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
1114+ inShift = bitsleft > 0;
1115+ } else {
1116+ *out++ = (char) ch;
1117+ }
1118+ } else {
1119+ if (!SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
1120+ *out++ = B64(charsleft << (6-bitsleft));
1121+ charsleft = 0;
1122+ bitsleft = 0;
1123+ /* Characters not in the BASE64 set implicitly unshift the sequence
1124+ so no '-' is required, except if the character is itself a '-' */
1125+ if (B64CHAR(ch) || ch == '-') {
1126+ *out++ = '-';
1127+ }
1128+ inShift = 0;
1129+ *out++ = (char) ch;
1130+ } else {
1131+ bitsleft += 16;
1132+ charsleft = (charsleft << 16) | ch;
1133+ /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
1134+
1135+ /* If the next character is special then we dont' need to terminate
1136+ the shift sequence. If the next character is not a BASE64 character
1137+ or '-' then the shift sequence will be terminated implicitly and we
1138+ don't have to insert a '-'. */
1139+
1140+ if (bitsleft == 0) {
1141+ if (i + 1 < size) {
1142+ Py_UNICODE ch2 = s[i+1];
1143+
1144+ if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) {
1145+
1146+ } else if (B64CHAR(ch2) || ch2 == '-') {
1147+ *out++ = '-';
1148+ inShift = 0;
1149+ } else {
1150+ inShift = 0;
1151+ }
1152+
1153+ }
1154+ else {
1155+ *out++ = '-';
1156+ inShift = 0;
1157+ }
1158+ }
1159+ }
1160+ }
1161+ }
1162+ if (bitsleft) {
1163+ *out++= B64(charsleft << (6-bitsleft) );
1164+ *out++ = '-';
1165+ }
1166+
1167+ _PyString_Resize(&v, out - start);
1168+ return v;
1169+}
1170+
1171+#undef SPECIAL
1172+#undef B64
1173+#undef B64CHAR
1174+#undef UB64
1175+#undef ENCODE
1176+#undef DECODE
1177+
1178+/* --- UTF-8 Codec -------------------------------------------------------- */
1179+
1180+static
1181+char utf8_code_length[256] = {
1182+ /* Map UTF-8 encoded prefix byte to sequence length. zero means
1183+ illegal prefix. see RFC 2279 for details */
1184+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1185+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1186+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1187+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1188+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1189+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1190+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1191+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1192+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1193+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1194+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1195+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1196+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1197+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1198+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
1199+ 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
1200+};
1201+
1202+PyObject *PyUnicode_DecodeUTF8(const char *s,
1203+ Py_ssize_t size,
1204+ const char *errors)
1205+{
1206+ return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
1207+}
1208+
1209+PyObject *PyUnicode_DecodeUTF8Stateful(const char *s,
1210+ Py_ssize_t size,
1211+ const char *errors,
1212+ Py_ssize_t *consumed)
1213+{
1214+ const char *starts = s;
1215+ int n;
1216+ Py_ssize_t startinpos;
1217+ Py_ssize_t endinpos;
1218+ Py_ssize_t outpos;
1219+ const char *e;
1220+ PyUnicodeObject *unicode;
1221+ Py_UNICODE *p;
1222+ const char *errmsg = "";
1223+ PyObject *errorHandler = NULL;
1224+ PyObject *exc = NULL;
1225+
1226+ /* Note: size will always be longer than the resulting Unicode
1227+ character count */
1228+ unicode = _PyUnicode_New(size);
1229+ if (!unicode)
1230+ return NULL;
1231+ if (size == 0) {
1232+ if (consumed)
1233+ *consumed = 0;
1234+ return (PyObject *)unicode;
1235+ }
1236+
1237+ /* Unpack UTF-8 encoded data */
1238+ p = unicode->str;
1239+ e = s + size;
1240+
1241+ while (s < e) {
1242+ Py_UCS4 ch = (unsigned char)*s;
1243+
1244+ if (ch < 0x80) {
1245+ *p++ = (Py_UNICODE)ch;
1246+ s++;
1247+ continue;
1248+ }
1249+
1250+ n = utf8_code_length[ch];
1251+
1252+ if (s + n > e) {
1253+ if (consumed)
1254+ break;
1255+ else {
1256+ errmsg = "unexpected end of data";
1257+ startinpos = s-starts;
1258+ endinpos = size;
1259+ goto utf8Error;
1260+ }
1261+ }
1262+
1263+ switch (n) {
1264+
1265+ case 0:
1266+ errmsg = "unexpected code byte";
1267+ startinpos = s-starts;
1268+ endinpos = startinpos+1;
1269+ goto utf8Error;
1270+
1271+ case 1:
1272+ errmsg = "internal error";
1273+ startinpos = s-starts;
1274+ endinpos = startinpos+1;
1275+ goto utf8Error;
1276+
1277+ case 2:
1278+ if ((s[1] & 0xc0) != 0x80) {
1279+ errmsg = "invalid data";
1280+ startinpos = s-starts;
1281+ endinpos = startinpos+2;
1282+ goto utf8Error;
1283+ }
1284+ ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);
1285+ if (ch < 0x80) {
1286+ startinpos = s-starts;
1287+ endinpos = startinpos+2;
1288+ errmsg = "illegal encoding";
1289+ goto utf8Error;
1290+ }
1291+ else
1292+ *p++ = (Py_UNICODE)ch;
1293+ break;
1294+
1295+ case 3:
1296+ if ((s[1] & 0xc0) != 0x80 ||
1297+ (s[2] & 0xc0) != 0x80) {
1298+ errmsg = "invalid data";
1299+ startinpos = s-starts;
1300+ endinpos = startinpos+3;
1301+ goto utf8Error;
1302+ }
1303+ ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
1304+ if (ch < 0x0800) {
1305+ /* Note: UTF-8 encodings of surrogates are considered
1306+ legal UTF-8 sequences;
1307+
1308+ XXX For wide builds (UCS-4) we should probably try
1309+ to recombine the surrogates into a single code
1310+ unit.
1311+ */
1312+ errmsg = "illegal encoding";
1313+ startinpos = s-starts;
1314+ endinpos = startinpos+3;
1315+ goto utf8Error;
1316+ }
1317+ else
1318+ *p++ = (Py_UNICODE)ch;
1319+ break;
1320+
1321+ case 4:
1322+ if ((s[1] & 0xc0) != 0x80 ||
1323+ (s[2] & 0xc0) != 0x80 ||
1324+ (s[3] & 0xc0) != 0x80) {
1325+ errmsg = "invalid data";
1326+ startinpos = s-starts;
1327+ endinpos = startinpos+4;
1328+ goto utf8Error;
1329+ }
1330+ ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +
1331+ ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);
1332+ /* validate and convert to UTF-16 */
1333+ if ((ch < 0x10000) /* minimum value allowed for 4
1334+ byte encoding */
1335+ || (ch > 0x10ffff)) /* maximum value allowed for
1336+ UTF-16 */
1337+ {
1338+ errmsg = "illegal encoding";
1339+ startinpos = s-starts;
1340+ endinpos = startinpos+4;
1341+ goto utf8Error;
1342+ }
1343+#ifdef Py_UNICODE_WIDE
1344+ *p++ = (Py_UNICODE)ch;
1345+#else
1346+ /* compute and append the two surrogates: */
1347+
1348+ /* translate from 10000..10FFFF to 0..FFFF */
1349+ ch -= 0x10000;
1350+
1351+ /* high surrogate = top 10 bits added to D800 */
1352+ *p++ = (Py_UNICODE)(0xD800 + (ch >> 10));
1353+
1354+ /* low surrogate = bottom 10 bits added to DC00 */
1355+ *p++ = (Py_UNICODE)(0xDC00 + (ch & 0x03FF));
1356+#endif
1357+ break;
1358+
1359+ default:
1360+ /* Other sizes are only needed for UCS-4 */
1361+ errmsg = "unsupported Unicode code range";
1362+ startinpos = s-starts;
1363+ endinpos = startinpos+n;
1364+ goto utf8Error;
1365+ }
1366+ s += n;
1367+ continue;
1368+
1369+ utf8Error:
1370+ outpos = p-PyUnicode_AS_UNICODE(unicode);
1371+ if (unicode_decode_call_errorhandler(
1372+ errors, &errorHandler,
1373+ "utf8", errmsg,
1374+ starts, size, &startinpos, &endinpos, &exc, &s,
1375+ (PyObject **)&unicode, &outpos, &p))
1376+ goto onError;
1377+ }
1378+ if (consumed)
1379+ *consumed = s-starts;
1380+
1381+ /* Adjust length */
1382+ if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0)
1383+ goto onError;
1384+
1385+ Py_XDECREF(errorHandler);
1386+ Py_XDECREF(exc);
1387+ return (PyObject *)unicode;
1388+
1389+onError:
1390+ Py_XDECREF(errorHandler);
1391+ Py_XDECREF(exc);
1392+ Py_DECREF(unicode);
1393+ return NULL;
1394+}
1395+
1396+/* Allocation strategy: if the string is short, convert into a stack buffer
1397+ and allocate exactly as much space needed at the end. Else allocate the
1398+ maximum possible needed (4 result bytes per Unicode character), and return
1399+ the excess memory at the end.
1400+*/
1401+PyObject *
1402+PyUnicode_EncodeUTF8(const Py_UNICODE *s,
1403+ Py_ssize_t size,
1404+ const char *errors)
1405+{
1406+#define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */
1407+
1408+ Py_ssize_t i; /* index into s of next input byte */
1409+ PyObject *v; /* result string object */
1410+ char *p; /* next free byte in output buffer */
1411+ Py_ssize_t nallocated; /* number of result bytes allocated */
1412+ Py_ssize_t nneeded; /* number of result bytes needed */
1413+ char stackbuf[MAX_SHORT_UNICHARS * 4];
1414+
1415+ assert(s != NULL);
1416+ assert(size >= 0);
1417+
1418+ if (size <= MAX_SHORT_UNICHARS) {
1419+ /* Write into the stack buffer; nallocated can't overflow.
1420+ * At the end, we'll allocate exactly as much heap space as it
1421+ * turns out we need.
1422+ */
1423+ nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);
1424+ v = NULL; /* will allocate after we're done */
1425+ p = stackbuf;
1426+ }
1427+ else {
1428+ /* Overallocate on the heap, and give the excess back at the end. */
1429+ nallocated = size * 4;
1430+ if (nallocated / 4 != size) /* overflow! */
1431+ return PyErr_NoMemory();
1432+ v = PyString_FromStringAndSize(NULL, nallocated);
1433+ if (v == NULL)
1434+ return NULL;
1435+ p = PyString_AS_STRING(v);
1436+ }
1437+
1438+ for (i = 0; i < size;) {
1439+ Py_UCS4 ch = s[i++];
1440+
1441+ if (ch < 0x80)
1442+ /* Encode ASCII */
1443+ *p++ = (char) ch;
1444+
1445+ else if (ch < 0x0800) {
1446+ /* Encode Latin-1 */
1447+ *p++ = (char)(0xc0 | (ch >> 6));
1448+ *p++ = (char)(0x80 | (ch & 0x3f));
1449+ }
1450+ else {
1451+ /* Encode UCS2 Unicode ordinals */
1452+ if (ch < 0x10000) {
1453+ /* Special case: check for high surrogate */
1454+ if (0xD800 <= ch && ch <= 0xDBFF && i != size) {
1455+ Py_UCS4 ch2 = s[i];
1456+ /* Check for low surrogate and combine the two to
1457+ form a UCS4 value */
1458+ if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
1459+ ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;
1460+ i++;
1461+ goto encodeUCS4;
1462+ }
1463+ /* Fall through: handles isolated high surrogates */
1464+ }
1465+ *p++ = (char)(0xe0 | (ch >> 12));
1466+ *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
1467+ *p++ = (char)(0x80 | (ch & 0x3f));
1468+ continue;
1469+ }
1470+encodeUCS4:
1471+ /* Encode UCS4 Unicode ordinals */
1472+ *p++ = (char)(0xf0 | (ch >> 18));
1473+ *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
1474+ *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
1475+ *p++ = (char)(0x80 | (ch & 0x3f));
1476+ }
1477+ }
1478+
1479+ if (v == NULL) {
1480+ /* This was stack allocated. */
1481+ nneeded = p - stackbuf;
1482+ assert(nneeded <= nallocated);
1483+ v = PyString_FromStringAndSize(stackbuf, nneeded);
1484+ }
1485+ else {
1486+ /* Cut back to size actually needed. */
1487+ nneeded = p - PyString_AS_STRING(v);
1488+ assert(nneeded <= nallocated);
1489+ _PyString_Resize(&v, nneeded);
1490+ }
1491+ return v;
1492+
1493+#undef MAX_SHORT_UNICHARS
1494+}
1495+
1496+PyObject *PyUnicode_AsUTF8String(PyObject *unicode)
1497+{
1498+ if (!PyUnicode_Check(unicode)) {
1499+ PyErr_BadArgument();
1500+ return NULL;
1501+ }
1502+ return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
1503+ PyUnicode_GET_SIZE(unicode),
1504+ NULL);
1505+}
1506+
1507+/* --- UTF-16 Codec ------------------------------------------------------- */
1508+
1509+PyObject *
1510+PyUnicode_DecodeUTF16(const char *s,
1511+ Py_ssize_t size,
1512+ const char *errors,
1513+ int *byteorder)
1514+{
1515+ return PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder, NULL);
1516+}
1517+
1518+PyObject *
1519+PyUnicode_DecodeUTF16Stateful(const char *s,
1520+ Py_ssize_t size,
1521+ const char *errors,
1522+ int *byteorder,
1523+ Py_ssize_t *consumed)
1524+{
1525+ const char *starts = s;
1526+ Py_ssize_t startinpos;
1527+ Py_ssize_t endinpos;
1528+ Py_ssize_t outpos;
1529+ PyUnicodeObject *unicode;
1530+ Py_UNICODE *p;
1531+ const unsigned char *q, *e;
1532+ int bo = 0; /* assume native ordering by default */
1533+ const char *errmsg = "";
1534+ /* Offsets from q for retrieving byte pairs in the right order. */
1535+#ifdef BYTEORDER_IS_LITTLE_ENDIAN
1536+ int ihi = 1, ilo = 0;
1537+#else
1538+ int ihi = 0, ilo = 1;
1539+#endif
1540+ PyObject *errorHandler = NULL;
1541+ PyObject *exc = NULL;
1542+
1543+ /* Note: size will always be longer than the resulting Unicode
1544+ character count */
1545+ unicode = _PyUnicode_New(size);
1546+ if (!unicode)
1547+ return NULL;
1548+ if (size == 0)
1549+ return (PyObject *)unicode;
1550+
1551+ /* Unpack UTF-16 encoded data */
1552+ p = unicode->str;
1553+ q = (unsigned char *)s;
1554+ e = q + size;
1555+
1556+ if (byteorder)
1557+ bo = *byteorder;
1558+
1559+ /* Check for BOM marks (U+FEFF) in the input and adjust current
1560+ byte order setting accordingly. In native mode, the leading BOM
1561+ mark is skipped, in all other modes, it is copied to the output
1562+ stream as-is (giving a ZWNBSP character). */
1563+ if (bo == 0) {
1564+ if (size >= 2) {
1565+ const Py_UNICODE bom = (q[ihi] << 8) | q[ilo];
1566+#ifdef BYTEORDER_IS_LITTLE_ENDIAN
1567+ if (bom == 0xFEFF) {
1568+ q += 2;
1569+ bo = -1;
1570+ }
1571+ else if (bom == 0xFFFE) {
1572+ q += 2;
1573+ bo = 1;
1574+ }
1575+#else
1576+ if (bom == 0xFEFF) {
1577+ q += 2;
1578+ bo = 1;
1579+ }
1580+ else if (bom == 0xFFFE) {
1581+ q += 2;
1582+ bo = -1;
1583+ }
1584+#endif
1585+ }
1586+ }
1587+
1588+ if (bo == -1) {
1589+ /* force LE */
1590+ ihi = 1;
1591+ ilo = 0;
1592+ }
1593+ else if (bo == 1) {
1594+ /* force BE */
1595+ ihi = 0;
1596+ ilo = 1;
1597+ }
1598+
1599+ while (q < e) {
1600+ Py_UNICODE ch;
1601+ /* remaining bytes at the end? (size should be even) */
1602+ if (e-q<2) {
1603+ if (consumed)
1604+ break;
1605+ errmsg = "truncated data";
1606+ startinpos = ((const char *)q)-starts;
1607+ endinpos = ((const char *)e)-starts;
1608+ goto utf16Error;
1609+ /* The remaining input chars are ignored if the callback
1610+ chooses to skip the input */
1611+ }
1612+ ch = (q[ihi] << 8) | q[ilo];
1613+
1614+ q += 2;
1615+
1616+ if (ch < 0xD800 || ch > 0xDFFF) {
1617+ *p++ = ch;
1618+ continue;
1619+ }
1620+
1621+ /* UTF-16 code pair: */
1622+ if (q >= e) {
1623+ errmsg = "unexpected end of data";
1624+ startinpos = (((const char *)q)-2)-starts;
1625+ endinpos = ((const char *)e)-starts;
1626+ goto utf16Error;
1627+ }
1628+ if (0xD800 <= ch && ch <= 0xDBFF) {
1629+ Py_UNICODE ch2 = (q[ihi] << 8) | q[ilo];
1630+ q += 2;
1631+ if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
1632+#ifndef Py_UNICODE_WIDE
1633+ *p++ = ch;
1634+ *p++ = ch2;
1635+#else
1636+ *p++ = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
1637+#endif
1638+ continue;
1639+ }
1640+ else {
1641+ errmsg = "illegal UTF-16 surrogate";
1642+ startinpos = (((const char *)q)-4)-starts;
1643+ endinpos = startinpos+2;
1644+ goto utf16Error;
1645+ }
1646+
1647+ }
1648+ errmsg = "illegal encoding";
1649+ startinpos = (((const char *)q)-2)-starts;
1650+ endinpos = startinpos+2;
1651+ /* Fall through to report the error */
1652+
1653+ utf16Error:
1654+ outpos = p-PyUnicode_AS_UNICODE(unicode);
1655+ if (unicode_decode_call_errorhandler(
1656+ errors, &errorHandler,
1657+ "utf16", errmsg,
1658+ starts, size, &startinpos, &endinpos, &exc, (const char **)&q,
1659+ (PyObject **)&unicode, &outpos, &p))
1660+ goto onError;
1661+ }
1662+
1663+ if (byteorder)
1664+ *byteorder = bo;
1665+
1666+ if (consumed)
1667+ *consumed = (const char *)q-starts;
1668+
1669+ /* Adjust length */
1670+ if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0)
1671+ goto onError;
1672+
1673+ Py_XDECREF(errorHandler);
1674+ Py_XDECREF(exc);
1675+ return (PyObject *)unicode;
1676+
1677+onError:
1678+ Py_DECREF(unicode);
1679+ Py_XDECREF(errorHandler);
1680+ Py_XDECREF(exc);
1681+ return NULL;
1682+}
1683+
1684+PyObject *
1685+PyUnicode_EncodeUTF16(const Py_UNICODE *s,
1686+ Py_ssize_t size,
1687+ const char *errors,
1688+ int byteorder)
1689+{
1690+ PyObject *v;
1691+ unsigned char *p;
1692+#ifdef Py_UNICODE_WIDE
1693+ int i, pairs;
1694+#else
1695+ const int pairs = 0;
1696+#endif
1697+ /* Offsets from p for storing byte pairs in the right order. */
1698+#ifdef BYTEORDER_IS_LITTLE_ENDIAN
1699+ int ihi = 1, ilo = 0;
1700+#else
1701+ int ihi = 0, ilo = 1;
1702+#endif
1703+
1704+#define STORECHAR(CH) \
1705+ do { \
1706+ p[ihi] = ((CH) >> 8) & 0xff; \
1707+ p[ilo] = (CH) & 0xff; \
1708+ p += 2; \
1709+ } while(0)
1710+
1711+#ifdef Py_UNICODE_WIDE
1712+ for (i = pairs = 0; i < size; i++)
1713+ if (s[i] >= 0x10000)
1714+ pairs++;
1715+#endif
1716+ v = PyString_FromStringAndSize(NULL,
1717+ 2 * (size + pairs + (byteorder == 0)));
1718+ if (v == NULL)
1719+ return NULL;
1720+
1721+ p = (unsigned char *)PyString_AS_STRING(v);
1722+ if (byteorder == 0)
1723+ STORECHAR(0xFEFF);
1724+ if (size == 0)
1725+ return v;
1726+
1727+ if (byteorder == -1) {
1728+ /* force LE */
1729+ ihi = 1;
1730+ ilo = 0;
1731+ }
1732+ else if (byteorder == 1) {
1733+ /* force BE */
1734+ ihi = 0;
1735+ ilo = 1;
1736+ }
1737+
1738+ while (size-- > 0) {
1739+ Py_UNICODE ch = *s++;
1740+ Py_UNICODE ch2 = 0;
1741+#ifdef Py_UNICODE_WIDE
1742+ if (ch >= 0x10000) {
1743+ ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF);
1744+ ch = 0xD800 | ((ch-0x10000) >> 10);
1745+ }
1746+#endif
1747+ STORECHAR(ch);
1748+ if (ch2)
1749+ STORECHAR(ch2);
1750+ }
1751+ return v;
1752+#undef STORECHAR
1753+}
1754+
1755+PyObject *PyUnicode_AsUTF16String(PyObject *unicode)
1756+{
1757+ if (!PyUnicode_Check(unicode)) {
1758+ PyErr_BadArgument();
1759+ return NULL;
1760+ }
1761+ return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(unicode),
1762+ PyUnicode_GET_SIZE(unicode),
1763+ NULL,
1764+ 0);
1765+}
1766+
1767+/* --- Unicode Escape Codec ----------------------------------------------- */
1768+
1769+static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
1770+
1771+PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
1772+ Py_ssize_t size,
1773+ const char *errors)
1774+{
1775+ const char *starts = s;
1776+ Py_ssize_t startinpos;
1777+ Py_ssize_t endinpos;
1778+ Py_ssize_t outpos;
1779+ int i;
1780+ PyUnicodeObject *v;
1781+ Py_UNICODE *p;
1782+ const char *end;
1783+ char* message;
1784+ Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */
1785+ PyObject *errorHandler = NULL;
1786+ PyObject *exc = NULL;
1787+
1788+ /* Escaped strings will always be longer than the resulting
1789+ Unicode string, so we start with size here and then reduce the
1790+ length after conversion to the true value.
1791+ (but if the error callback returns a long replacement string
1792+ we'll have to allocate more space) */
1793+ v = _PyUnicode_New(size);
1794+ if (v == NULL)
1795+ goto onError;
1796+ if (size == 0)
1797+ return (PyObject *)v;
1798+
1799+ p = PyUnicode_AS_UNICODE(v);
1800+ end = s + size;
1801+
1802+ while (s < end) {
1803+ unsigned char c;
1804+ Py_UNICODE x;
1805+ int digits;
1806+
1807+ /* Non-escape characters are interpreted as Unicode ordinals */
1808+ if (*s != '\\') {
1809+ *p++ = (unsigned char) *s++;
1810+ continue;
1811+ }
1812+
1813+ startinpos = s-starts;
1814+ /* \ - Escapes */
1815+ s++;
1816+ switch (*s++) {
1817+
1818+ /* \x escapes */
1819+ case '\n': break;
1820+ case '\\': *p++ = '\\'; break;
1821+ case '\'': *p++ = '\''; break;
1822+ case '\"': *p++ = '\"'; break;
1823+ case 'b': *p++ = '\b'; break;
1824+ case 'f': *p++ = '\014'; break; /* FF */
1825+ case 't': *p++ = '\t'; break;
1826+ case 'n': *p++ = '\n'; break;
1827+ case 'r': *p++ = '\r'; break;
1828+ case 'v': *p++ = '\013'; break; /* VT */
1829+ case 'a': *p++ = '\007'; break; /* BEL, not classic C */
1830+
1831+ /* \OOO (octal) escapes */
1832+ case '0': case '1': case '2': case '3':
1833+ case '4': case '5': case '6': case '7':
1834+ x = s[-1] - '0';
1835+ if ('0' <= *s && *s <= '7') {
1836+ x = (x<<3) + *s++ - '0';
1837+ if ('0' <= *s && *s <= '7')
1838+ x = (x<<3) + *s++ - '0';
1839+ }
1840+ *p++ = x;
1841+ break;
1842+
1843+ /* hex escapes */
1844+ /* \xXX */
1845+ case 'x':
1846+ digits = 2;
1847+ message = "truncated \\xXX escape";
1848+ goto hexescape;
1849+
1850+ /* \uXXXX */
1851+ case 'u':
1852+ digits = 4;
1853+ message = "truncated \\uXXXX escape";
1854+ goto hexescape;
1855+
1856+ /* \UXXXXXXXX */
1857+ case 'U':
1858+ digits = 8;
1859+ message = "truncated \\UXXXXXXXX escape";
1860+ hexescape:
1861+ chr = 0;
1862+ outpos = p-PyUnicode_AS_UNICODE(v);
1863+ if (s+digits>end) {
1864+ endinpos = size;
1865+ if (unicode_decode_call_errorhandler(
1866+ errors, &errorHandler,
1867+ "unicodeescape", "end of string in escape sequence",
1868+ starts, size, &startinpos, &endinpos, &exc, &s,
1869+ (PyObject **)&v, &outpos, &p))
1870+ goto onError;
1871+ goto nextByte;
1872+ }
1873+ for (i = 0; i < digits; ++i) {
1874+ c = (unsigned char) s[i];
1875+ if (!isxdigit(c)) {
1876+ endinpos = (s+i+1)-starts;
1877+ if (unicode_decode_call_errorhandler(
1878+ errors, &errorHandler,
1879+ "unicodeescape", message,
1880+ starts, size, &startinpos, &endinpos, &exc, &s,
1881+ (PyObject **)&v, &outpos, &p))
1882+ goto onError;
1883+ goto nextByte;
1884+ }
1885+ chr = (chr<<4) & ~0xF;
1886+ if (c >= '0' && c <= '9')
1887+ chr += c - '0';
1888+ else if (c >= 'a' && c <= 'f')
1889+ chr += 10 + c - 'a';
1890+ else
1891+ chr += 10 + c - 'A';
1892+ }
1893+ s += i;
1894+ if (chr == 0xffffffff && PyErr_Occurred())
1895+ /* _decoding_error will have already written into the
1896+ target buffer. */
1897+ break;
1898+ store:
1899+ /* when we get here, chr is a 32-bit unicode character */
1900+ if (chr <= 0xffff)
1901+ /* UCS-2 character */
1902+ *p++ = (Py_UNICODE) chr;
1903+ else if (chr <= 0x10ffff) {
1904+ /* UCS-4 character. Either store directly, or as
1905+ surrogate pair. */
1906+#ifdef Py_UNICODE_WIDE
1907+ *p++ = chr;
1908+#else
1909+ chr -= 0x10000L;
1910+ *p++ = 0xD800 + (Py_UNICODE) (chr >> 10);
1911+ *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF);
1912+#endif
1913+ } else {
1914+ endinpos = s-starts;
1915+ outpos = p-PyUnicode_AS_UNICODE(v);
1916+ if (unicode_decode_call_errorhandler(
1917+ errors, &errorHandler,
1918+ "unicodeescape", "illegal Unicode character",
1919+ starts, size, &startinpos, &endinpos, &exc, &s,
1920+ (PyObject **)&v, &outpos, &p))
1921+ goto onError;
1922+ }
1923+ break;
1924+
1925+ /* \N{name} */
1926+ case 'N':
1927+ message = "malformed \\N character escape";
1928+ if (ucnhash_CAPI == NULL) {
1929+ /* load the unicode data module */
1930+ PyObject *m, *api;
1931+ m = PyImport_ImportModule("unicodedata");
1932+ if (m == NULL)
1933+ goto ucnhashError;
1934+ api = PyObject_GetAttrString(m, "ucnhash_CAPI");
1935+ Py_DECREF(m);
1936+ if (api == NULL)
1937+ goto ucnhashError;
1938+ ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCObject_AsVoidPtr(api);
1939+ Py_DECREF(api);
1940+ if (ucnhash_CAPI == NULL)
1941+ goto ucnhashError;
1942+ }
1943+ if (*s == '{') {
1944+ const char *start = s+1;
1945+ /* look for the closing brace */
1946+ while (*s != '}' && s < end)
1947+ s++;
1948+ if (s > start && s < end && *s == '}') {
1949+ /* found a name. look it up in the unicode database */
1950+ message = "unknown Unicode character name";
1951+ s++;
1952+ if (ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), &chr))
1953+ goto store;
1954+ }
1955+ }
1956+ endinpos = s-starts;
1957+ outpos = p-PyUnicode_AS_UNICODE(v);
1958+ if (unicode_decode_call_errorhandler(
1959+ errors, &errorHandler,
1960+ "unicodeescape", message,
1961+ starts, size, &startinpos, &endinpos, &exc, &s,
1962+ (PyObject **)&v, &outpos, &p))
1963+ goto onError;
1964+ break;
1965+
1966+ default:
1967+ if (s > end) {
1968+ message = "\\ at end of string";
1969+ s--;
1970+ endinpos = s-starts;
1971+ outpos = p-PyUnicode_AS_UNICODE(v);
1972+ if (unicode_decode_call_errorhandler(
1973+ errors, &errorHandler,
1974+ "unicodeescape", message,
1975+ starts, size, &startinpos, &endinpos, &exc, &s,
1976+ (PyObject **)&v, &outpos, &p))
1977+ goto onError;
1978+ }
1979+ else {
1980+ *p++ = '\\';
1981+ *p++ = (unsigned char)s[-1];
1982+ }
1983+ break;
1984+ }
1985+ nextByte:
1986+ ;
1987+ }
1988+ if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
1989+ goto onError;
1990+ Py_XDECREF(errorHandler);
1991+ Py_XDECREF(exc);
1992+ return (PyObject *)v;
1993+
1994+ucnhashError:
1995+ PyErr_SetString(
1996+ PyExc_UnicodeError,
1997+ "\\N escapes not supported (can't load unicodedata module)"
1998+ );
1999+ Py_XDECREF(v);
2000+ Py_XDECREF(errorHandler);
2001+ Py_XDECREF(exc);
2002+ return NULL;
2003+
2004+onError:
2005+ Py_XDECREF(v);
2006+ Py_XDECREF(errorHandler);
2007+ Py_XDECREF(exc);
2008+ return NULL;
2009+}
2010+
2011+/* Return a Unicode-Escape string version of the Unicode object.
2012+
2013+ If quotes is true, the string is enclosed in u"" or u'' quotes as
2014+ appropriate.
2015+
2016+*/
2017+
2018+Py_LOCAL_INLINE(const Py_UNICODE *) findchar(const Py_UNICODE *s,
2019+ Py_ssize_t size,
2020+ Py_UNICODE ch)
2021+{
2022+ /* like wcschr, but doesn't stop at NULL characters */
2023+
2024+ while (size-- > 0) {
2025+ if (*s == ch)
2026+ return s;
2027+ s++;
2028+ }
2029+
2030+ return NULL;
2031+}
2032+
2033+static
2034+PyObject *unicodeescape_string(const Py_UNICODE *s,
2035+ Py_ssize_t size,
2036+ int quotes)
2037+{
2038+ PyObject *repr;
2039+ char *p;
2040+
2041+ static const char *hexdigit = "0123456789abcdef";
2042+
2043+ /* Initial allocation is based on the longest-possible unichr
2044+ escape.
2045+
2046+ In wide (UTF-32) builds '\U00xxxxxx' is 10 chars per source
2047+ unichr, so in this case it's the longest unichr escape. In
2048+ narrow (UTF-16) builds this is five chars per source unichr
2049+ since there are two unichrs in the surrogate pair, so in narrow
2050+ (UTF-16) builds it's not the longest unichr escape.
2051+
2052+ In wide or narrow builds '\uxxxx' is 6 chars per source unichr,
2053+ so in the narrow (UTF-16) build case it's the longest unichr
2054+ escape.
2055+ */
2056+
2057+ repr = PyString_FromStringAndSize(NULL,
2058+ 2
2059+#ifdef Py_UNICODE_WIDE
2060+ + 10*size
2061+#else
2062+ + 6*size
2063+#endif
2064+ + 1);
2065+ if (repr == NULL)
2066+ return NULL;
2067+
2068+ p = PyString_AS_STRING(repr);
2069+
2070+ if (quotes) {
2071+ *p++ = 'u';
2072+ *p++ = (findchar(s, size, '\'') &&
2073+ !findchar(s, size, '"')) ? '"' : '\'';
2074+ }
2075+ while (size-- > 0) {
2076+ Py_UNICODE ch = *s++;
2077+
2078+ /* Escape quotes and backslashes */
2079+ if ((quotes &&
2080+ ch == (Py_UNICODE) PyString_AS_STRING(repr)[1]) || ch == '\\') {
2081+ *p++ = '\\';
2082+ *p++ = (char) ch;
2083+ continue;
2084+ }
2085+
2086+#ifdef Py_UNICODE_WIDE
2087+ /* Map 21-bit characters to '\U00xxxxxx' */
2088+ else if (ch >= 0x10000) {
2089+ *p++ = '\\';
2090+ *p++ = 'U';
2091+ *p++ = hexdigit[(ch >> 28) & 0x0000000F];
2092+ *p++ = hexdigit[(ch >> 24) & 0x0000000F];
2093+ *p++ = hexdigit[(ch >> 20) & 0x0000000F];
2094+ *p++ = hexdigit[(ch >> 16) & 0x0000000F];
2095+ *p++ = hexdigit[(ch >> 12) & 0x0000000F];
2096+ *p++ = hexdigit[(ch >> 8) & 0x0000000F];
2097+ *p++ = hexdigit[(ch >> 4) & 0x0000000F];
2098+ *p++ = hexdigit[ch & 0x0000000F];
2099+ continue;
2100+ }
2101+#else
2102+ /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
2103+ else if (ch >= 0xD800 && ch < 0xDC00) {
2104+ Py_UNICODE ch2;
2105+ Py_UCS4 ucs;
2106+
2107+ ch2 = *s++;
2108+ size--;
2109+ if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
2110+ ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
2111+ *p++ = '\\';
2112+ *p++ = 'U';
2113+ *p++ = hexdigit[(ucs >> 28) & 0x0000000F];
2114+ *p++ = hexdigit[(ucs >> 24) & 0x0000000F];
2115+ *p++ = hexdigit[(ucs >> 20) & 0x0000000F];
2116+ *p++ = hexdigit[(ucs >> 16) & 0x0000000F];
2117+ *p++ = hexdigit[(ucs >> 12) & 0x0000000F];
2118+ *p++ = hexdigit[(ucs >> 8) & 0x0000000F];
2119+ *p++ = hexdigit[(ucs >> 4) & 0x0000000F];
2120+ *p++ = hexdigit[ucs & 0x0000000F];
2121+ continue;
2122+ }
2123+ /* Fall through: isolated surrogates are copied as-is */
2124+ s--;
2125+ size++;
2126+ }
2127+#endif
2128+
2129+ /* Map 16-bit characters to '\uxxxx' */
2130+ if (ch >= 256) {
2131+ *p++ = '\\';
2132+ *p++ = 'u';
2133+ *p++ = hexdigit[(ch >> 12) & 0x000F];
2134+ *p++ = hexdigit[(ch >> 8) & 0x000F];
2135+ *p++ = hexdigit[(ch >> 4) & 0x000F];
2136+ *p++ = hexdigit[ch & 0x000F];
2137+ }
2138+
2139+ /* Map special whitespace to '\t', \n', '\r' */
2140+ else if (ch == '\t') {
2141+ *p++ = '\\';
2142+ *p++ = 't';
2143+ }
2144+ else if (ch == '\n') {
2145+ *p++ = '\\';
2146+ *p++ = 'n';
2147+ }
2148+ else if (ch == '\r') {
2149+ *p++ = '\\';
2150+ *p++ = 'r';
2151+ }
2152+
2153+ /* Map non-printable US ASCII to '\xhh' */
2154+ else if (ch < ' ' || ch >= 0x7F) {
2155+ *p++ = '\\';
2156+ *p++ = 'x';
2157+ *p++ = hexdigit[(ch >> 4) & 0x000F];
2158+ *p++ = hexdigit[ch & 0x000F];
2159+ }
2160+
2161+ /* Copy everything else as-is */
2162+ else
2163+ *p++ = (char) ch;
2164+ }
2165+ if (quotes)
2166+ *p++ = PyString_AS_STRING(repr)[1];
2167+
2168+ *p = '\0';
2169+ _PyString_Resize(&repr, p - PyString_AS_STRING(repr));
2170+ return repr;
2171+}
2172+
2173+PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
2174+ Py_ssize_t size)
2175+{
2176+ return unicodeescape_string(s, size, 0);
2177+}
2178+
2179+PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
2180+{
2181+ if (!PyUnicode_Check(unicode)) {
2182+ PyErr_BadArgument();
2183+ return NULL;
2184+ }
2185+ return PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
2186+ PyUnicode_GET_SIZE(unicode));
2187+}
2188+
2189+/* --- Raw Unicode Escape Codec ------------------------------------------- */
2190+
2191+PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s,
2192+ Py_ssize_t size,
2193+ const char *errors)
2194+{
2195+ const char *starts = s;
2196+ Py_ssize_t startinpos;
2197+ Py_ssize_t endinpos;
2198+ Py_ssize_t outpos;
2199+ PyUnicodeObject *v;
2200+ Py_UNICODE *p;
2201+ const char *end;
2202+ const char *bs;
2203+ PyObject *errorHandler = NULL;
2204+ PyObject *exc = NULL;
2205+
2206+ /* Escaped strings will always be longer than the resulting
2207+ Unicode string, so we start with size here and then reduce the
2208+ length after conversion to the true value. (But decoding error
2209+ handler might have to resize the string) */
2210+ v = _PyUnicode_New(size);
2211+ if (v == NULL)
2212+ goto onError;
2213+ if (size == 0)
2214+ return (PyObject *)v;
2215+ p = PyUnicode_AS_UNICODE(v);
2216+ end = s + size;
2217+ while (s < end) {
2218+ unsigned char c;
2219+ Py_UCS4 x;
2220+ int i;
2221+ int count;
2222+
2223+ /* Non-escape characters are interpreted as Unicode ordinals */
2224+ if (*s != '\\') {
2225+ *p++ = (unsigned char)*s++;
2226+ continue;
2227+ }
2228+ startinpos = s-starts;
2229+
2230+ /* \u-escapes are only interpreted iff the number of leading
2231+ backslashes if odd */
2232+ bs = s;
2233+ for (;s < end;) {
2234+ if (*s != '\\')
2235+ break;
2236+ *p++ = (unsigned char)*s++;
2237+ }
2238+ if (((s - bs) & 1) == 0 ||
2239+ s >= end ||
2240+ (*s != 'u' && *s != 'U')) {
2241+ continue;
2242+ }
2243+ p--;
2244+ count = *s=='u' ? 4 : 8;
2245+ s++;
2246+
2247+ /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */
2248+ outpos = p-PyUnicode_AS_UNICODE(v);
2249+ for (x = 0, i = 0; i < count; ++i, ++s) {
2250+ c = (unsigned char)*s;
2251+ if (!isxdigit(c)) {
2252+ endinpos = s-starts;
2253+ if (unicode_decode_call_errorhandler(
2254+ errors, &errorHandler,
2255+ "rawunicodeescape", "truncated \\uXXXX",
2256+ starts, size, &startinpos, &endinpos, &exc, &s,
2257+ (PyObject **)&v, &outpos, &p))
2258+ goto onError;
2259+ goto nextByte;
2260+ }
2261+ x = (x<<4) & ~0xF;
2262+ if (c >= '0' && c <= '9')
2263+ x += c - '0';
2264+ else if (c >= 'a' && c <= 'f')
2265+ x += 10 + c - 'a';
2266+ else
2267+ x += 10 + c - 'A';
2268+ }
2269+#ifndef Py_UNICODE_WIDE
2270+ if (x > 0x10000) {
2271+ if (unicode_decode_call_errorhandler(
2272+ errors, &errorHandler,
2273+ "rawunicodeescape", "\\Uxxxxxxxx out of range",
2274+ starts, size, &startinpos, &endinpos, &exc, &s,
2275+ (PyObject **)&v, &outpos, &p))
2276+ goto onError;
2277+ }
2278+#endif
2279+ *p++ = x;
2280+ nextByte:
2281+ ;
2282+ }
2283+ if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
2284+ goto onError;
2285+ Py_XDECREF(errorHandler);
2286+ Py_XDECREF(exc);
2287+ return (PyObject *)v;
2288+
2289+ onError:
2290+ Py_XDECREF(v);
2291+ Py_XDECREF(errorHandler);
2292+ Py_XDECREF(exc);
2293+ return NULL;
2294+}
2295+
2296+PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
2297+ Py_ssize_t size)
2298+{
2299+ PyObject *repr;
2300+ char *p;
2301+ char *q;
2302+
2303+ static const char *hexdigit = "0123456789abcdef";
2304+
2305+#ifdef Py_UNICODE_WIDE
2306+ repr = PyString_FromStringAndSize(NULL, 10 * size);
2307+#else
2308+ repr = PyString_FromStringAndSize(NULL, 6 * size);
2309+#endif
2310+ if (repr == NULL)
2311+ return NULL;
2312+ if (size == 0)
2313+ return repr;
2314+
2315+ p = q = PyString_AS_STRING(repr);
2316+ while (size-- > 0) {
2317+ Py_UNICODE ch = *s++;
2318+#ifdef Py_UNICODE_WIDE
2319+ /* Map 32-bit characters to '\Uxxxxxxxx' */
2320+ if (ch >= 0x10000) {
2321+ *p++ = '\\';
2322+ *p++ = 'U';
2323+ *p++ = hexdigit[(ch >> 28) & 0xf];
2324+ *p++ = hexdigit[(ch >> 24) & 0xf];
2325+ *p++ = hexdigit[(ch >> 20) & 0xf];
2326+ *p++ = hexdigit[(ch >> 16) & 0xf];
2327+ *p++ = hexdigit[(ch >> 12) & 0xf];
2328+ *p++ = hexdigit[(ch >> 8) & 0xf];
2329+ *p++ = hexdigit[(ch >> 4) & 0xf];
2330+ *p++ = hexdigit[ch & 15];
2331+ }
2332+ else
2333+#endif
2334+ /* Map 16-bit characters to '\uxxxx' */
2335+ if (ch >= 256) {
2336+ *p++ = '\\';
2337+ *p++ = 'u';
2338+ *p++ = hexdigit[(ch >> 12) & 0xf];
2339+ *p++ = hexdigit[(ch >> 8) & 0xf];
2340+ *p++ = hexdigit[(ch >> 4) & 0xf];
2341+ *p++ = hexdigit[ch & 15];
2342+ }
2343+ /* Copy everything else as-is */
2344+ else
2345+ *p++ = (char) ch;
2346+ }
2347+ *p = '\0';
2348+ _PyString_Resize(&repr, p - q);
2349+ return repr;
2350+}
2351+
2352+PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
2353+{
2354+ if (!PyUnicode_Check(unicode)) {
2355+ PyErr_BadArgument();
2356+ return NULL;
2357+ }
2358+ return PyUnicode_EncodeRawUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
2359+ PyUnicode_GET_SIZE(unicode));
2360+}
2361+
2362+/* --- Unicode Internal Codec ------------------------------------------- */
2363+
2364+PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s,
2365+ Py_ssize_t size,
2366+ const char *errors)
2367+{
2368+ const char *starts = s;
2369+ Py_ssize_t startinpos;
2370+ Py_ssize_t endinpos;
2371+ Py_ssize_t outpos;
2372+ PyUnicodeObject *v;
2373+ Py_UNICODE *p;
2374+ const char *end;
2375+ const char *reason;
2376+ PyObject *errorHandler = NULL;
2377+ PyObject *exc = NULL;
2378+
2379+#ifdef Py_UNICODE_WIDE
2380+ Py_UNICODE unimax = PyUnicode_GetMax();
2381+#endif
2382+
2383+ /* XXX overflow detection missing */
2384+ v = _PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE);
2385+ if (v == NULL)
2386+ goto onError;
2387+ if (PyUnicode_GetSize((PyObject *)v) == 0)
2388+ return (PyObject *)v;
2389+ p = PyUnicode_AS_UNICODE(v);
2390+ end = s + size;
2391+
2392+ while (s < end) {
2393+ memcpy(p, s, sizeof(Py_UNICODE));
2394+ /* We have to sanity check the raw data, otherwise doom looms for
2395+ some malformed UCS-4 data. */
2396+ if (
2397+ #ifdef Py_UNICODE_WIDE
2398+ *p > unimax || *p < 0 ||
2399+ #endif
2400+ end-s < Py_UNICODE_SIZE
2401+ )
2402+ {
2403+ startinpos = s - starts;
2404+ if (end-s < Py_UNICODE_SIZE) {
2405+ endinpos = end-starts;
2406+ reason = "truncated input";
2407+ }
2408+ else {
2409+ endinpos = s - starts + Py_UNICODE_SIZE;
2410+ reason = "illegal code point (> 0x10FFFF)";
2411+ }
2412+ outpos = p - PyUnicode_AS_UNICODE(v);
2413+ if (unicode_decode_call_errorhandler(
2414+ errors, &errorHandler,
2415+ "unicode_internal", reason,
2416+ starts, size, &startinpos, &endinpos, &exc, &s,
2417+ (PyObject **)&v, &outpos, &p)) {
2418+ goto onError;
2419+ }
2420+ }
2421+ else {
2422+ p++;
2423+ s += Py_UNICODE_SIZE;
2424+ }
2425+ }
2426+
2427+ if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
2428+ goto onError;
2429+ Py_XDECREF(errorHandler);
2430+ Py_XDECREF(exc);
2431+ return (PyObject *)v;
2432+
2433+ onError:
2434+ Py_XDECREF(v);
2435+ Py_XDECREF(errorHandler);
2436+ Py_XDECREF(exc);
2437+ return NULL;
2438+}
2439+
2440+/* --- Latin-1 Codec ------------------------------------------------------ */
2441+
2442+PyObject *PyUnicode_DecodeLatin1(const char *s,
2443+ Py_ssize_t size,
2444+ const char *errors)
2445+{
2446+ PyUnicodeObject *v;
2447+ Py_UNICODE *p;
2448+
2449+ /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */
2450+ if (size == 1) {
2451+ Py_UNICODE r = *(unsigned char*)s;
2452+ return PyUnicode_FromUnicode(&r, 1);
2453+ }
2454+
2455+ v = _PyUnicode_New(size);
2456+ if (v == NULL)
2457+ goto onError;
2458+ if (size == 0)
2459+ return (PyObject *)v;
2460+ p = PyUnicode_AS_UNICODE(v);
2461+ while (size-- > 0)
2462+ *p++ = (unsigned char)*s++;
2463+ return (PyObject *)v;
2464+
2465+ onError:
2466+ Py_XDECREF(v);
2467+ return NULL;
2468+}
2469+
2470+/* create or adjust a UnicodeEncodeError */
2471+static void make_encode_exception(PyObject **exceptionObject,
2472+ const char *encoding,
2473+ const Py_UNICODE *unicode, Py_ssize_t size,
2474+ Py_ssize_t startpos, Py_ssize_t endpos,
2475+ const char *reason)
2476+{
2477+ if (*exceptionObject == NULL) {
2478+ *exceptionObject = PyUnicodeEncodeError_Create(
2479+ encoding, unicode, size, startpos, endpos, reason);
2480+ }
2481+ else {
2482+ if (PyUnicodeEncodeError_SetStart(*exceptionObject, startpos))
2483+ goto onError;
2484+ if (PyUnicodeEncodeError_SetEnd(*exceptionObject, endpos))
2485+ goto onError;
2486+ if (PyUnicodeEncodeError_SetReason(*exceptionObject, reason))
2487+ goto onError;
2488+ return;
2489+ onError:
2490+ Py_DECREF(*exceptionObject);
2491+ *exceptionObject = NULL;
2492+ }
2493+}
2494+
2495+/* raises a UnicodeEncodeError */
2496+static void raise_encode_exception(PyObject **exceptionObject,
2497+ const char *encoding,
2498+ const Py_UNICODE *unicode, Py_ssize_t size,
2499+ Py_ssize_t startpos, Py_ssize_t endpos,
2500+ const char *reason)
2501+{
2502+ make_encode_exception(exceptionObject,
2503+ encoding, unicode, size, startpos, endpos, reason);
2504+ if (*exceptionObject != NULL)
2505+ PyCodec_StrictErrors(*exceptionObject);
2506+}
2507+
2508+/* error handling callback helper:
2509+ build arguments, call the callback and check the arguments,
2510+ put the result into newpos and return the replacement string, which
2511+ has to be freed by the caller */
2512+static PyObject *unicode_encode_call_errorhandler(const char *errors,
2513+ PyObject **errorHandler,
2514+ const char *encoding, const char *reason,
2515+ const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
2516+ Py_ssize_t startpos, Py_ssize_t endpos,
2517+ Py_ssize_t *newpos)
2518+{
2519+ static char *argparse = "O!n;encoding error handler must return (unicode, int) tuple";
2520+
2521+ PyObject *restuple;
2522+ PyObject *resunicode;
2523+
2524+ if (*errorHandler == NULL) {
2525+ *errorHandler = PyCodec_LookupError(errors);
2526+ if (*errorHandler == NULL)
2527+ return NULL;
2528+ }
2529+
2530+ make_encode_exception(exceptionObject,
2531+ encoding, unicode, size, startpos, endpos, reason);
2532+ if (*exceptionObject == NULL)
2533+ return NULL;
2534+
2535+ restuple = PyObject_CallFunctionObjArgs(
2536+ *errorHandler, *exceptionObject, NULL);
2537+ if (restuple == NULL)
2538+ return NULL;
2539+ if (!PyTuple_Check(restuple)) {
2540+ PyErr_Format(PyExc_TypeError, &argparse[4]);
2541+ Py_DECREF(restuple);
2542+ return NULL;
2543+ }
2544+ if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type,
2545+ &resunicode, newpos)) {
2546+ Py_DECREF(restuple);
2547+ return NULL;
2548+ }
2549+ if (*newpos<0)
2550+ *newpos = size+*newpos;
2551+ if (*newpos<0 || *newpos>size) {
2552+ PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
2553+ Py_DECREF(restuple);
2554+ return NULL;
2555+ }
2556+ Py_INCREF(resunicode);
2557+ Py_DECREF(restuple);
2558+ return resunicode;
2559+}
2560+
2561+static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
2562+ Py_ssize_t size,
2563+ const char *errors,
2564+ int limit)
2565+{
2566+ /* output object */
2567+ PyObject *res;
2568+ /* pointers to the beginning and end+1 of input */
2569+ const Py_UNICODE *startp = p;
2570+ const Py_UNICODE *endp = p + size;
2571+ /* pointer to the beginning of the unencodable characters */
2572+ /* const Py_UNICODE *badp = NULL; */
2573+ /* pointer into the output */
2574+ char *str;
2575+ /* current output position */
2576+ Py_ssize_t respos = 0;
2577+ Py_ssize_t ressize;
2578+ const char *encoding = (limit == 256) ? "latin-1" : "ascii";
2579+ const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
2580+ PyObject *errorHandler = NULL;
2581+ PyObject *exc = NULL;
2582+ /* the following variable is used for caching string comparisons
2583+ * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
2584+ int known_errorHandler = -1;
2585+
2586+ /* allocate enough for a simple encoding without
2587+ replacements, if we need more, we'll resize */
2588+ res = PyString_FromStringAndSize(NULL, size);
2589+ if (res == NULL)
2590+ goto onError;
2591+ if (size == 0)
2592+ return res;
2593+ str = PyString_AS_STRING(res);
2594+ ressize = size;
2595+
2596+ while (p<endp) {
2597+ Py_UNICODE c = *p;
2598+
2599+ /* can we encode this? */
2600+ if (c<limit) {
2601+ /* no overflow check, because we know that the space is enough */
2602+ *str++ = (char)c;
2603+ ++p;
2604+ }
2605+ else {
2606+ Py_ssize_t unicodepos = p-startp;
2607+ Py_ssize_t requiredsize;
2608+ PyObject *repunicode;
2609+ Py_ssize_t repsize;
2610+ Py_ssize_t newpos;
2611+ Py_ssize_t respos;
2612+ Py_UNICODE *uni2;
2613+ /* startpos for collecting unencodable chars */
2614+ const Py_UNICODE *collstart = p;
2615+ const Py_UNICODE *collend = p;
2616+ /* find all unecodable characters */
2617+ while ((collend < endp) && ((*collend)>=limit))
2618+ ++collend;
2619+ /* cache callback name lookup (if not done yet, i.e. it's the first error) */
2620+ if (known_errorHandler==-1) {
2621+ if ((errors==NULL) || (!strcmp(errors, "strict")))
2622+ known_errorHandler = 1;
2623+ else if (!strcmp(errors, "replace"))
2624+ known_errorHandler = 2;
2625+ else if (!strcmp(errors, "ignore"))
2626+ known_errorHandler = 3;
2627+ else if (!strcmp(errors, "xmlcharrefreplace"))
2628+ known_errorHandler = 4;
2629+ else
2630+ known_errorHandler = 0;
2631+ }
2632+ switch (known_errorHandler) {
2633+ case 1: /* strict */
2634+ raise_encode_exception(&exc, encoding, startp, size, collstart-startp, collend-startp, reason);
2635+ goto onError;
2636+ case 2: /* replace */
2637+ while (collstart++<collend)
2638+ *str++ = '?'; /* fall through */
2639+ case 3: /* ignore */
2640+ p = collend;
2641+ break;
2642+ case 4: /* xmlcharrefreplace */
2643+ respos = str-PyString_AS_STRING(res);
2644+ /* determine replacement size (temporarily (mis)uses p) */
2645+ for (p = collstart, repsize = 0; p < collend; ++p) {
2646+ if (*p<10)
2647+ repsize += 2+1+1;
2648+ else if (*p<100)
2649+ repsize += 2+2+1;
2650+ else if (*p<1000)
2651+ repsize += 2+3+1;
2652+ else if (*p<10000)
2653+ repsize += 2+4+1;
2654+#ifndef Py_UNICODE_WIDE
2655+ else
2656+ repsize += 2+5+1;
2657+#else
2658+ else if (*p<100000)
2659+ repsize += 2+5+1;
2660+ else if (*p<1000000)
2661+ repsize += 2+6+1;
2662+ else
2663+ repsize += 2+7+1;
2664+#endif
2665+ }
2666+ requiredsize = respos+repsize+(endp-collend);
2667+ if (requiredsize > ressize) {
2668+ if (requiredsize<2*ressize)
2669+ requiredsize = 2*ressize;
2670+ if (_PyString_Resize(&res, requiredsize))
2671+ goto onError;
2672+ str = PyString_AS_STRING(res) + respos;
2673+ ressize = requiredsize;
2674+ }
2675+ /* generate replacement (temporarily (mis)uses p) */
2676+ for (p = collstart; p < collend; ++p) {
2677+ str += sprintf(str, "&#%d;", (int)*p);
2678+ }
2679+ p = collend;
2680+ break;
2681+ default:
2682+ repunicode = unicode_encode_call_errorhandler(errors, &errorHandler,
2683+ encoding, reason, startp, size, &exc,
2684+ collstart-startp, collend-startp, &newpos);
2685+ if (repunicode == NULL)
2686+ goto onError;
2687+ /* need more space? (at least enough for what we
2688+ have+the replacement+the rest of the string, so
2689+ we won't have to check space for encodable characters) */
2690+ respos = str-PyString_AS_STRING(res);
2691+ repsize = PyUnicode_GET_SIZE(repunicode);
2692+ requiredsize = respos+repsize+(endp-collend);
2693+ if (requiredsize > ressize) {
2694+ if (requiredsize<2*ressize)
2695+ requiredsize = 2*ressize;
2696+ if (_PyString_Resize(&res, requiredsize)) {
2697+ Py_DECREF(repunicode);
2698+ goto onError;
2699+ }
2700+ str = PyString_AS_STRING(res) + respos;
2701+ ressize = requiredsize;
2702+ }
2703+ /* check if there is anything unencodable in the replacement
2704+ and copy it to the output */
2705+ for (uni2 = PyUnicode_AS_UNICODE(repunicode);repsize-->0; ++uni2, ++str) {
2706+ c = *uni2;
2707+ if (c >= limit) {
2708+ raise_encode_exception(&exc, encoding, startp, size,
2709+ unicodepos, unicodepos+1, reason);
2710+ Py_DECREF(repunicode);
2711+ goto onError;
2712+ }
2713+ *str = (char)c;
2714+ }
2715+ p = startp + newpos;
2716+ Py_DECREF(repunicode);
2717+ }
2718+ }
2719+ }
2720+ /* Resize if we allocated to much */
2721+ respos = str-PyString_AS_STRING(res);
2722+ if (respos<ressize)
2723+ /* If this falls res will be NULL */
2724+ _PyString_Resize(&res, respos);
2725+ Py_XDECREF(errorHandler);
2726+ Py_XDECREF(exc);
2727+ return res;
2728+
2729+ onError:
2730+ Py_XDECREF(res);
2731+ Py_XDECREF(errorHandler);
2732+ Py_XDECREF(exc);
2733+ return NULL;
2734+}
2735+
2736+PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p,
2737+ Py_ssize_t size,
2738+ const char *errors)
2739+{
2740+ return unicode_encode_ucs1(p, size, errors, 256);
2741+}
2742+
2743+PyObject *PyUnicode_AsLatin1String(PyObject *unicode)
2744+{
2745+ if (!PyUnicode_Check(unicode)) {
2746+ PyErr_BadArgument();
2747+ return NULL;
2748+ }
2749+ return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode),
2750+ PyUnicode_GET_SIZE(unicode),
2751+ NULL);
2752+}
2753+
2754+/* --- 7-bit ASCII Codec -------------------------------------------------- */
2755+
2756+PyObject *PyUnicode_DecodeASCII(const char *s,
2757+ Py_ssize_t size,
2758+ const char *errors)
2759+{
2760+ const char *starts = s;
2761+ PyUnicodeObject *v;
2762+ Py_UNICODE *p;
2763+ Py_ssize_t startinpos;
2764+ Py_ssize_t endinpos;
2765+ Py_ssize_t outpos;
2766+ const char *e;
2767+ PyObject *errorHandler = NULL;
2768+ PyObject *exc = NULL;
2769+
2770+ /* ASCII is equivalent to the first 128 ordinals in Unicode. */
2771+ if (size == 1 && *(unsigned char*)s < 128) {
2772+ Py_UNICODE r = *(unsigned char*)s;
2773+ return PyUnicode_FromUnicode(&r, 1);
2774+ }
2775+
2776+ v = _PyUnicode_New(size);
2777+ if (v == NULL)
2778+ goto onError;
2779+ if (size == 0)
2780+ return (PyObject *)v;
2781+ p = PyUnicode_AS_UNICODE(v);
2782+ e = s + size;
2783+ while (s < e) {
2784+ register unsigned char c = (unsigned char)*s;
2785+ if (c < 128) {
2786+ *p++ = c;
2787+ ++s;
2788+ }
2789+ else {
2790+ startinpos = s-starts;
2791+ endinpos = startinpos + 1;
2792+ outpos = p - (Py_UNICODE *)PyUnicode_AS_UNICODE(v);
2793+ if (unicode_decode_call_errorhandler(
2794+ errors, &errorHandler,
2795+ "ascii", "ordinal not in range(128)",
2796+ starts, size, &startinpos, &endinpos, &exc, &s,
2797+ (PyObject **)&v, &outpos, &p))
2798+ goto onError;
2799+ }
2800+ }
2801+ if (p - PyUnicode_AS_UNICODE(v) < PyString_GET_SIZE(v))
2802+ if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
2803+ goto onError;
2804+ Py_XDECREF(errorHandler);
2805+ Py_XDECREF(exc);
2806+ return (PyObject *)v;
2807+
2808+ onError:
2809+ Py_XDECREF(v);
2810+ Py_XDECREF(errorHandler);
2811+ Py_XDECREF(exc);
2812+ return NULL;
2813+}
2814+
2815+PyObject *PyUnicode_EncodeASCII(const Py_UNICODE *p,
2816+ Py_ssize_t size,
2817+ const char *errors)
2818+{
2819+ return unicode_encode_ucs1(p, size, errors, 128);
2820+}
2821+
2822+PyObject *PyUnicode_AsASCIIString(PyObject *unicode)
2823+{
2824+ if (!PyUnicode_Check(unicode)) {
2825+ PyErr_BadArgument();
2826+ return NULL;
2827+ }
2828+ return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode),
2829+ PyUnicode_GET_SIZE(unicode),
2830+ NULL);
2831+}
2832+
2833+#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
2834+
2835+/* --- MBCS codecs for Windows -------------------------------------------- */
2836+
2837+#if SIZEOF_INT < SIZEOF_SSIZE_T
2838+#define NEED_RETRY
2839+#endif
2840+
2841+/* XXX This code is limited to "true" double-byte encodings, as
2842+ a) it assumes an incomplete character consists of a single byte, and
2843+ b) IsDBCSLeadByte (probably) does not work for non-DBCS multi-byte
2844+ encodings, see IsDBCSLeadByteEx documentation. */
2845+
2846+static int is_dbcs_lead_byte(const char *s, int offset)
2847+{
2848+ const char *curr = s + offset;
2849+
2850+ if (IsDBCSLeadByte(*curr)) {
2851+ const char *prev = CharPrev(s, curr);
2852+ return (prev == curr) || !IsDBCSLeadByte(*prev) || (curr - prev == 2);
2853+ }
2854+ return 0;
2855+}
2856+
2857+/*
2858+ * Decode MBCS string into unicode object. If 'final' is set, converts
2859+ * trailing lead-byte too. Returns consumed size if succeed, -1 otherwise.
2860+ */
2861+static int decode_mbcs(PyUnicodeObject **v,
2862+ const char *s, /* MBCS string */
2863+ int size, /* sizeof MBCS string */
2864+ int final)
2865+{
2866+ Py_UNICODE *p;
2867+ Py_ssize_t n = 0;
2868+ int usize = 0;
2869+
2870+ assert(size >= 0);
2871+
2872+ /* Skip trailing lead-byte unless 'final' is set */
2873+ if (!final && size >= 1 && is_dbcs_lead_byte(s, size - 1))
2874+ --size;
2875+
2876+ /* First get the size of the result */
2877+ if (size > 0) {
2878+ usize = MultiByteToWideChar(CP_ACP, 0, s, size, NULL, 0);
2879+ if (usize == 0) {
2880+ PyErr_SetFromWindowsErrWithFilename(0, NULL);
2881+ return -1;
2882+ }
2883+ }
2884+
2885+ if (*v == NULL) {
2886+ /* Create unicode object */
2887+ *v = _PyUnicode_New(usize);
2888+ if (*v == NULL)
2889+ return -1;
2890+ }
2891+ else {
2892+ /* Extend unicode object */
2893+ n = PyUnicode_GET_SIZE(*v);
2894+ if (_PyUnicode_Resize(v, n + usize) < 0)
2895+ return -1;
2896+ }
2897+
2898+ /* Do the conversion */
2899+ if (size > 0) {
2900+ p = PyUnicode_AS_UNICODE(*v) + n;
2901+ if (0 == MultiByteToWideChar(CP_ACP, 0, s, size, p, usize)) {
2902+ PyErr_SetFromWindowsErrWithFilename(0, NULL);
2903+ return -1;
2904+ }
2905+ }
2906+
2907+ return size;
2908+}
2909+
2910+PyObject *PyUnicode_DecodeMBCSStateful(const char *s,
2911+ Py_ssize_t size,
2912+ const char *errors,
2913+ Py_ssize_t *consumed)
2914+{
2915+ PyUnicodeObject *v = NULL;
2916+ int done;
2917+
2918+ if (consumed)
2919+ *consumed = 0;
2920+
2921+#ifdef NEED_RETRY
2922+ retry:
2923+ if (size > INT_MAX)
2924+ done = decode_mbcs(&v, s, INT_MAX, 0);
2925+ else
2926+#endif
2927+ done = decode_mbcs(&v, s, (int)size, !consumed);
2928+
2929+ if (done < 0) {
2930+ Py_XDECREF(v);
2931+ return NULL;
2932+ }
2933+
2934+ if (consumed)
2935+ *consumed += done;
2936+
2937+#ifdef NEED_RETRY
2938+ if (size > INT_MAX) {
2939+ s += done;
2940+ size -= done;
2941+ goto retry;
2942+ }
2943+#endif
2944+
2945+ return (PyObject *)v;
2946+}
2947+
2948+PyObject *PyUnicode_DecodeMBCS(const char *s,
2949+ Py_ssize_t size,
2950+ const char *errors)
2951+{
2952+ return PyUnicode_DecodeMBCSStateful(s, size, errors, NULL);
2953+}
2954+
2955+/*
2956+ * Convert unicode into string object (MBCS).
2957+ * Returns 0 if succeed, -1 otherwise.
2958+ */
2959+static int encode_mbcs(PyObject **repr,
2960+ const Py_UNICODE *p, /* unicode */
2961+ int size) /* size of unicode */
2962+{
2963+ int mbcssize = 0;
2964+ Py_ssize_t n = 0;
2965+
2966+ assert(size >= 0);
2967+
2968+ /* First get the size of the result */
2969+ if (size > 0) {
2970+ mbcssize = WideCharToMultiByte(CP_ACP, 0, p, size, NULL, 0, NULL, NULL);
2971+ if (mbcssize == 0) {
2972+ PyErr_SetFromWindowsErrWithFilename(0, NULL);
2973+ return -1;
2974+ }
2975+ }
2976+
2977+ if (*repr == NULL) {
2978+ /* Create string object */
2979+ *repr = PyString_FromStringAndSize(NULL, mbcssize);
2980+ if (*repr == NULL)
2981+ return -1;
2982+ }
2983+ else {
2984+ /* Extend string object */
2985+ n = PyString_Size(*repr);
2986+ if (_PyString_Resize(repr, n + mbcssize) < 0)
2987+ return -1;
2988+ }
2989+
2990+ /* Do the conversion */
2991+ if (size > 0) {
2992+ char *s = PyString_AS_STRING(*repr) + n;
2993+ if (0 == WideCharToMultiByte(CP_ACP, 0, p, size, s, mbcssize, NULL, NULL)) {
2994+ PyErr_SetFromWindowsErrWithFilename(0, NULL);
2995+ return -1;
2996+ }
2997+ }
2998+
2999+ return 0;
3000+}
3001+
3002+PyObject *PyUnicode_EncodeMBCS(const Py_UNICODE *p,
3003+ Py_ssize_t size,
3004+ const char *errors)
3005+{
3006+ PyObject *repr = NULL;
3007+ int ret;
3008+
3009+#ifdef NEED_RETRY
3010+ retry:
3011+ if (size > INT_MAX)
3012+ ret = encode_mbcs(&repr, p, INT_MAX);
3013+ else
3014+#endif
3015+ ret = encode_mbcs(&repr, p, (int)size);
3016+
3017+ if (ret < 0) {
3018+ Py_XDECREF(repr);
3019+ return NULL;
3020+ }
3021+
3022+#ifdef NEED_RETRY
3023+ if (size > INT_MAX) {
3024+ p += INT_MAX;
3025+ size -= INT_MAX;
3026+ goto retry;
3027+ }
3028+#endif
3029+
3030+ return repr;
3031+}
3032+
3033+PyObject *PyUnicode_AsMBCSString(PyObject *unicode)
3034+{
3035+ if (!PyUnicode_Check(unicode)) {
3036+ PyErr_BadArgument();
3037+ return NULL;
3038+ }
3039+ return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode),
3040+ PyUnicode_GET_SIZE(unicode),
3041+ NULL);
3042+}
3043+
3044+#undef NEED_RETRY
3045+
3046+#endif /* MS_WINDOWS */
3047+
3048+/* --- Character Mapping Codec -------------------------------------------- */
3049+
3050+PyObject *PyUnicode_DecodeCharmap(const char *s,
3051+ Py_ssize_t size,
3052+ PyObject *mapping,
3053+ const char *errors)
3054+{
3055+ const char *starts = s;
3056+ Py_ssize_t startinpos;
3057+ Py_ssize_t endinpos;
3058+ Py_ssize_t outpos;
3059+ const char *e;
3060+ PyUnicodeObject *v;
3061+ Py_UNICODE *p;
3062+ Py_ssize_t extrachars = 0;
3063+ PyObject *errorHandler = NULL;
3064+ PyObject *exc = NULL;
3065+ Py_UNICODE *mapstring = NULL;
3066+ Py_ssize_t maplen = 0;
3067+
3068+ /* Default to Latin-1 */
3069+ if (mapping == NULL)
3070+ return PyUnicode_DecodeLatin1(s, size, errors);
3071+
3072+ v = _PyUnicode_New(size);
3073+ if (v == NULL)
3074+ goto onError;
3075+ if (size == 0)
3076+ return (PyObject *)v;
3077+ p = PyUnicode_AS_UNICODE(v);
3078+ e = s + size;
3079+ if (PyUnicode_CheckExact(mapping)) {
3080+ mapstring = PyUnicode_AS_UNICODE(mapping);
3081+ maplen = PyUnicode_GET_SIZE(mapping);
3082+ while (s < e) {
3083+ unsigned char ch = *s;
3084+ Py_UNICODE x = 0xfffe; /* illegal value */
3085+
3086+ if (ch < maplen)
3087+ x = mapstring[ch];
3088+
3089+ if (x == 0xfffe) {
3090+ /* undefined mapping */
3091+ outpos = p-PyUnicode_AS_UNICODE(v);
3092+ startinpos = s-starts;
3093+ endinpos = startinpos+1;
3094+ if (unicode_decode_call_errorhandler(
3095+ errors, &errorHandler,
3096+ "charmap", "character maps to <undefined>",
3097+ starts, size, &startinpos, &endinpos, &exc, &s,
3098+ (PyObject **)&v, &outpos, &p)) {
3099+ goto onError;
3100+ }
3101+ continue;
3102+ }
3103+ *p++ = x;
3104+ ++s;
3105+ }
3106+ }
3107+ else {
3108+ while (s < e) {
3109+ unsigned char ch = *s;
3110+ PyObject *w, *x;
3111+
3112+ /* Get mapping (char ordinal -> integer, Unicode char or None) */
3113+ w = PyInt_FromLong((long)ch);
3114+ if (w == NULL)
3115+ goto onError;
3116+ x = PyObject_GetItem(mapping, w);
3117+ Py_DECREF(w);
3118+ if (x == NULL) {
3119+ if (PyErr_ExceptionMatches(PyExc_LookupError)) {
3120+ /* No mapping found means: mapping is undefined. */
3121+ PyErr_Clear();
3122+ x = Py_None;
3123+ Py_INCREF(x);
3124+ } else
3125+ goto onError;
3126+ }
3127+
3128+ /* Apply mapping */
3129+ if (PyInt_Check(x)) {
3130+ long value = PyInt_AS_LONG(x);
3131+ if (value < 0 || value > 65535) {
3132+ PyErr_SetString(PyExc_TypeError,
3133+ "character mapping must be in range(65536)");
3134+ Py_DECREF(x);
3135+ goto onError;
3136+ }
3137+ *p++ = (Py_UNICODE)value;
3138+ }
3139+ else if (x == Py_None) {
3140+ /* undefined mapping */
3141+ outpos = p-PyUnicode_AS_UNICODE(v);
3142+ startinpos = s-starts;
3143+ endinpos = startinpos+1;
3144+ if (unicode_decode_call_errorhandler(
3145+ errors, &errorHandler,
3146+ "charmap", "character maps to <undefined>",
3147+ starts, size, &startinpos, &endinpos, &exc, &s,
3148+ (PyObject **)&v, &outpos, &p)) {
3149+ Py_DECREF(x);
3150+ goto onError;
3151+ }
3152+ Py_DECREF(x);
3153+ continue;
3154+ }
3155+ else if (PyUnicode_Check(x)) {
3156+ Py_ssize_t targetsize = PyUnicode_GET_SIZE(x);
3157+
3158+ if (targetsize == 1)
3159+ /* 1-1 mapping */
3160+ *p++ = *PyUnicode_AS_UNICODE(x);
3161+
3162+ else if (targetsize > 1) {
3163+ /* 1-n mapping */
3164+ if (targetsize > extrachars) {
3165+ /* resize first */
3166+ Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v);
3167+ Py_ssize_t needed = (targetsize - extrachars) + \
3168+ (targetsize << 2);
3169+ extrachars += needed;
3170+ /* XXX overflow detection missing */
3171+ if (_PyUnicode_Resize(&v,
3172+ PyUnicode_GET_SIZE(v) + needed) < 0) {
3173+ Py_DECREF(x);
3174+ goto onError;
3175+ }
3176+ p = PyUnicode_AS_UNICODE(v) + oldpos;
3177+ }
3178+ Py_UNICODE_COPY(p,
3179+ PyUnicode_AS_UNICODE(x),
3180+ targetsize);
3181+ p += targetsize;
3182+ extrachars -= targetsize;
3183+ }
3184+ /* 1-0 mapping: skip the character */
3185+ }
3186+ else {
3187+ /* wrong return value */
3188+ PyErr_SetString(PyExc_TypeError,
3189+ "character mapping must return integer, None or unicode");
3190+ Py_DECREF(x);
3191+ goto onError;
3192+ }
3193+ Py_DECREF(x);
3194+ ++s;
3195+ }
3196+ }
3197+ if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v))
3198+ if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
3199+ goto onError;
3200+ Py_XDECREF(errorHandler);
3201+ Py_XDECREF(exc);
3202+ return (PyObject *)v;
3203+
3204+ onError:
3205+ Py_XDECREF(errorHandler);
3206+ Py_XDECREF(exc);
3207+ Py_XDECREF(v);
3208+ return NULL;
3209+}
3210+
3211+/* Charmap encoding: the lookup table */
3212+
3213+struct encoding_map{
3214+ PyObject_HEAD
3215+ unsigned char level1[32];
3216+ int count2, count3;
3217+ unsigned char level23[1];
3218+};
3219+
3220+static PyObject*
3221+encoding_map_size(PyObject *obj, PyObject* args)
3222+{
3223+ struct encoding_map *map = (struct encoding_map*)obj;
3224+ return PyInt_FromLong(sizeof(*map) - 1 + 16*map->count2 +
3225+ 128*map->count3);
3226+}
3227+
3228+static PyMethodDef encoding_map_methods[] = {
3229+ {"size", encoding_map_size, METH_NOARGS,
3230+ PyDoc_STR("Return the size (in bytes) of this object") },
3231+ { 0 }
3232+};
3233+
3234+static void
3235+encoding_map_dealloc(PyObject* o)
3236+{
3237+ PyObject_FREE(o);
3238+}
3239+
3240+static PyTypeObject EncodingMapType = {
3241+ PyObject_HEAD_INIT(NULL)
3242+ 0, /*ob_size*/
3243+ "EncodingMap", /*tp_name*/
3244+ sizeof(struct encoding_map), /*tp_basicsize*/
3245+ 0, /*tp_itemsize*/
3246+ /* methods */
3247+ encoding_map_dealloc, /*tp_dealloc*/
3248+ 0, /*tp_print*/
3249+ 0, /*tp_getattr*/
3250+ 0, /*tp_setattr*/
3251+ 0, /*tp_compare*/
3252+ 0, /*tp_repr*/
3253+ 0, /*tp_as_number*/
3254+ 0, /*tp_as_sequence*/
3255+ 0, /*tp_as_mapping*/
3256+ 0, /*tp_hash*/
3257+ 0, /*tp_call*/
3258+ 0, /*tp_str*/
3259+ 0, /*tp_getattro*/
3260+ 0, /*tp_setattro*/
3261+ 0, /*tp_as_buffer*/
3262+ Py_TPFLAGS_DEFAULT, /*tp_flags*/
3263+ 0, /*tp_doc*/
3264+ 0, /*tp_traverse*/
3265+ 0, /*tp_clear*/
3266+ 0, /*tp_richcompare*/
3267+ 0, /*tp_weaklistoffset*/
3268+ 0, /*tp_iter*/
3269+ 0, /*tp_iternext*/
3270+ encoding_map_methods, /*tp_methods*/
3271+ 0, /*tp_members*/
3272+ 0, /*tp_getset*/
3273+ 0, /*tp_base*/
3274+ 0, /*tp_dict*/
3275+ 0, /*tp_descr_get*/
3276+ 0, /*tp_descr_set*/
3277+ 0, /*tp_dictoffset*/
3278+ 0, /*tp_init*/
3279+ 0, /*tp_alloc*/
3280+ 0, /*tp_new*/
3281+ 0, /*tp_free*/
3282+ 0, /*tp_is_gc*/
3283+};
3284+
3285+PyObject*
3286+PyUnicode_BuildEncodingMap(PyObject* string)
3287+{
3288+ Py_UNICODE *decode;
3289+ PyObject *result;
3290+ struct encoding_map *mresult;
3291+ int i;
3292+ int need_dict = 0;
3293+ unsigned char level1[32];
3294+ unsigned char level2[512];
3295+ unsigned char *mlevel1, *mlevel2, *mlevel3;
3296+ int count2 = 0, count3 = 0;
3297+
3298+ if (!PyUnicode_Check(string) || PyUnicode_GetSize(string) != 256) {
3299+ PyErr_BadArgument();
3300+ return NULL;
3301+ }
3302+ decode = PyUnicode_AS_UNICODE(string);
3303+ memset(level1, 0xFF, sizeof level1);
3304+ memset(level2, 0xFF, sizeof level2);
3305+
3306+ /* If there isn't a one-to-one mapping of NULL to \0,
3307+ or if there are non-BMP characters, we need to use
3308+ a mapping dictionary. */
3309+ if (decode[0] != 0)
3310+ need_dict = 1;
3311+ for (i = 1; i < 256; i++) {
3312+ int l1, l2;
3313+ if (decode[i] == 0
3314+ #ifdef Py_UNICODE_WIDE
3315+ || decode[i] > 0xFFFF
3316+ #endif
3317+ ) {
3318+ need_dict = 1;
3319+ break;
3320+ }
3321+ if (decode[i] == 0xFFFE)
3322+ /* unmapped character */
3323+ continue;
3324+ l1 = decode[i] >> 11;
3325+ l2 = decode[i] >> 7;
3326+ if (level1[l1] == 0xFF)
3327+ level1[l1] = count2++;
3328+ if (level2[l2] == 0xFF)
3329+ level2[l2] = count3++;
3330+ }
3331+
3332+ if (count2 >= 0xFF || count3 >= 0xFF)
3333+ need_dict = 1;
3334+
3335+ if (need_dict) {
3336+ PyObject *result = PyDict_New();
3337+ PyObject *key, *value;
3338+ if (!result)
3339+ return NULL;
3340+ for (i = 0; i < 256; i++) {
3341+ key = value = NULL;
3342+ key = PyInt_FromLong(decode[i]);
3343+ value = PyInt_FromLong(i);
3344+ if (!key || !value)
3345+ goto failed1;
3346+ if (PyDict_SetItem(result, key, value) == -1)
3347+ goto failed1;
3348+ Py_DECREF(key);
3349+ Py_DECREF(value);
3350+ }
3351+ return result;
3352+ failed1:
3353+ Py_XDECREF(key);
3354+ Py_XDECREF(value);
3355+ Py_DECREF(result);
3356+ return NULL;
3357+ }
3358+
3359+ /* Create a three-level trie */
3360+ result = PyObject_MALLOC(sizeof(struct encoding_map) +
3361+ 16*count2 + 128*count3 - 1);
3362+ if (!result)
3363+ return PyErr_NoMemory();
3364+ PyObject_Init(result, &EncodingMapType);
3365+ mresult = (struct encoding_map*)result;
3366+ mresult->count2 = count2;
3367+ mresult->count3 = count3;
3368+ mlevel1 = mresult->level1;
3369+ mlevel2 = mresult->level23;
3370+ mlevel3 = mresult->level23 + 16*count2;
3371+ memcpy(mlevel1, level1, 32);
3372+ memset(mlevel2, 0xFF, 16*count2);
3373+ memset(mlevel3, 0, 128*count3);
3374+ count3 = 0;
3375+ for (i = 1; i < 256; i++) {
3376+ int o1, o2, o3, i2, i3;
3377+ if (decode[i] == 0xFFFE)
3378+ /* unmapped character */
3379+ continue;
3380+ o1 = decode[i]>>11;
3381+ o2 = (decode[i]>>7) & 0xF;
3382+ i2 = 16*mlevel1[o1] + o2;
3383+ if (mlevel2[i2] == 0xFF)
3384+ mlevel2[i2] = count3++;
3385+ o3 = decode[i] & 0x7F;
3386+ i3 = 128*mlevel2[i2] + o3;
3387+ mlevel3[i3] = i;
3388+ }
3389+ return result;
3390+}
3391+
3392+static int
3393+encoding_map_lookup(Py_UNICODE c, PyObject *mapping)
3394+{
3395+ struct encoding_map *map = (struct encoding_map*)mapping;
3396+ int l1 = c>>11;
3397+ int l2 = (c>>7) & 0xF;
3398+ int l3 = c & 0x7F;
3399+ int i;
3400+
3401+#ifdef Py_UNICODE_WIDE
3402+ if (c > 0xFFFF) {
3403+ return -1;
3404+ }
3405+#endif
3406+ if (c == 0)
3407+ return 0;
3408+ /* level 1*/
3409+ i = map->level1[l1];
3410+ if (i == 0xFF) {
3411+ return -1;
3412+ }
3413+ /* level 2*/
3414+ i = map->level23[16*i+l2];
3415+ if (i == 0xFF) {
3416+ return -1;
3417+ }
3418+ /* level 3 */
3419+ i = map->level23[16*map->count2 + 128*i + l3];
3420+ if (i == 0) {
3421+ return -1;
3422+ }
3423+ return i;
3424+}
3425+
3426+/* Lookup the character ch in the mapping. If the character
3427+ can't be found, Py_None is returned (or NULL, if another
3428+ error occurred). */
3429+static PyObject *charmapencode_lookup(Py_UNICODE c, PyObject *mapping)
3430+{
3431+ PyObject *w = PyInt_FromLong((long)c);
3432+ PyObject *x;
3433+
3434+ if (w == NULL)
3435+ return NULL;
3436+ x = PyObject_GetItem(mapping, w);
3437+ Py_DECREF(w);
3438+ if (x == NULL) {
3439+ if (PyErr_ExceptionMatches(PyExc_LookupError)) {
3440+ /* No mapping found means: mapping is undefined. */
3441+ PyErr_Clear();
3442+ x = Py_None;
3443+ Py_INCREF(x);
3444+ return x;
3445+ } else
3446+ return NULL;
3447+ }
3448+ else if (x == Py_None)
3449+ return x;
3450+ else if (PyInt_Check(x)) {
3451+ long value = PyInt_AS_LONG(x);
3452+ if (value < 0 || value > 255) {
3453+ PyErr_SetString(PyExc_TypeError,
3454+ "character mapping must be in range(256)");
3455+ Py_DECREF(x);
3456+ return NULL;
3457+ }
3458+ return x;
3459+ }
3460+ else if (PyString_Check(x))
3461+ return x;
3462+ else {
3463+ /* wrong return value */
3464+ PyErr_SetString(PyExc_TypeError,
3465+ "character mapping must return integer, None or str");
3466+ Py_DECREF(x);
3467+ return NULL;
3468+ }
3469+}
3470+
3471+static int
3472+charmapencode_resize(PyObject **outobj, Py_ssize_t *outpos, Py_ssize_t requiredsize)
3473+{
3474+ Py_ssize_t outsize = PyString_GET_SIZE(*outobj);
3475+ /* exponentially overallocate to minimize reallocations */
3476+ if (requiredsize < 2*outsize)
3477+ requiredsize = 2*outsize;
3478+ if (_PyString_Resize(outobj, requiredsize)) {
3479+ return 0;
3480+ }
3481+ return 1;
3482+}
3483+
3484+typedef enum charmapencode_result {
3485+ enc_SUCCESS, enc_FAILED, enc_EXCEPTION
3486+}charmapencode_result;
3487+/* lookup the character, put the result in the output string and adjust
3488+ various state variables. Reallocate the output string if not enough
3489+ space is available. Return a new reference to the object that
3490+ was put in the output buffer, or Py_None, if the mapping was undefined
3491+ (in which case no character was written) or NULL, if a
3492+ reallocation error occurred. The caller must decref the result */
3493+static
3494+charmapencode_result charmapencode_output(Py_UNICODE c, PyObject *mapping,
3495+ PyObject **outobj, Py_ssize_t *outpos)
3496+{
3497+ PyObject *rep;
3498+ char *outstart;
3499+ Py_ssize_t outsize = PyString_GET_SIZE(*outobj);
3500+
3501+ if (mapping->ob_type == &EncodingMapType) {
3502+ int res = encoding_map_lookup(c, mapping);
3503+ Py_ssize_t requiredsize = *outpos+1;
3504+ if (res == -1)
3505+ return enc_FAILED;
3506+ if (outsize<requiredsize)
3507+ if (!charmapencode_resize(outobj, outpos, requiredsize))
3508+ return enc_EXCEPTION;
3509+ outstart = PyString_AS_STRING(*outobj);
3510+ outstart[(*outpos)++] = (char)res;
3511+ return enc_SUCCESS;
3512+ }
3513+
3514+ rep = charmapencode_lookup(c, mapping);
3515+ if (rep==NULL)
3516+ return enc_EXCEPTION;
3517+ else if (rep==Py_None) {
3518+ Py_DECREF(rep);
3519+ return enc_FAILED;
3520+ } else {
3521+ if (PyInt_Check(rep)) {
3522+ Py_ssize_t requiredsize = *outpos+1;
3523+ if (outsize<requiredsize)
3524+ if (!charmapencode_resize(outobj, outpos, requiredsize)) {
3525+ Py_DECREF(rep);
3526+ return enc_EXCEPTION;
3527+ }
3528+ outstart = PyString_AS_STRING(*outobj);
3529+ outstart[(*outpos)++] = (char)PyInt_AS_LONG(rep);
3530+ }
3531+ else {
3532+ const char *repchars = PyString_AS_STRING(rep);
3533+ Py_ssize_t repsize = PyString_GET_SIZE(rep);
3534+ Py_ssize_t requiredsize = *outpos+repsize;
3535+ if (outsize<requiredsize)
3536+ if (!charmapencode_resize(outobj, outpos, requiredsize)) {
3537+ Py_DECREF(rep);
3538+ return enc_EXCEPTION;
3539+ }
3540+ outstart = PyString_AS_STRING(*outobj);
3541+ memcpy(outstart + *outpos, repchars, repsize);
3542+ *outpos += repsize;
3543+ }
3544+ }
3545+ Py_DECREF(rep);
3546+ return enc_SUCCESS;
3547+}
3548+
3549+/* handle an error in PyUnicode_EncodeCharmap
3550+ Return 0 on success, -1 on error */
3551+static
3552+int charmap_encoding_error(
3553+ const Py_UNICODE *p, Py_ssize_t size, Py_ssize_t *inpos, PyObject *mapping,
3554+ PyObject **exceptionObject,
3555+ int *known_errorHandler, PyObject **errorHandler, const char *errors,
3556+ PyObject **res, Py_ssize_t *respos)
3557+{
3558+ PyObject *repunicode = NULL; /* initialize to prevent gcc warning */
3559+ Py_ssize_t repsize;
3560+ Py_ssize_t newpos;
3561+ Py_UNICODE *uni2;
3562+ /* startpos for collecting unencodable chars */
3563+ Py_ssize_t collstartpos = *inpos;
3564+ Py_ssize_t collendpos = *inpos+1;
3565+ Py_ssize_t collpos;
3566+ char *encoding = "charmap";
3567+ char *reason = "character maps to <undefined>";
3568+ charmapencode_result x;
3569+
3570+ /* find all unencodable characters */
3571+ while (collendpos < size) {
3572+ PyObject *rep;
3573+ if (mapping->ob_type == &EncodingMapType) {
3574+ int res = encoding_map_lookup(p[collendpos], mapping);
3575+ if (res != -1)
3576+ break;
3577+ ++collendpos;
3578+ continue;
3579+ }
3580+
3581+ rep = charmapencode_lookup(p[collendpos], mapping);
3582+ if (rep==NULL)
3583+ return -1;
3584+ else if (rep!=Py_None) {
3585+ Py_DECREF(rep);
3586+ break;
3587+ }
3588+ Py_DECREF(rep);
3589+ ++collendpos;
3590+ }
3591+ /* cache callback name lookup
3592+ * (if not done yet, i.e. it's the first error) */
3593+ if (*known_errorHandler==-1) {
3594+ if ((errors==NULL) || (!strcmp(errors, "strict")))
3595+ *known_errorHandler = 1;
3596+ else if (!strcmp(errors, "replace"))
3597+ *known_errorHandler = 2;
3598+ else if (!strcmp(errors, "ignore"))
3599+ *known_errorHandler = 3;
3600+ else if (!strcmp(errors, "xmlcharrefreplace"))
3601+ *known_errorHandler = 4;
3602+ else
3603+ *known_errorHandler = 0;
3604+ }
3605+ switch (*known_errorHandler) {
3606+ case 1: /* strict */
3607+ raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3608+ return -1;
3609+ case 2: /* replace */
3610+ for (collpos = collstartpos; collpos<collendpos; ++collpos) {
3611+ x = charmapencode_output('?', mapping, res, respos);
3612+ if (x==enc_EXCEPTION) {
3613+ return -1;
3614+ }
3615+ else if (x==enc_FAILED) {
3616+ raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3617+ return -1;
3618+ }
3619+ }
3620+ /* fall through */
3621+ case 3: /* ignore */
3622+ *inpos = collendpos;
3623+ break;
3624+ case 4: /* xmlcharrefreplace */
3625+ /* generate replacement (temporarily (mis)uses p) */
3626+ for (collpos = collstartpos; collpos < collendpos; ++collpos) {
3627+ char buffer[2+29+1+1];
3628+ char *cp;
3629+ sprintf(buffer, "&#%d;", (int)p[collpos]);
3630+ for (cp = buffer; *cp; ++cp) {
3631+ x = charmapencode_output(*cp, mapping, res, respos);
3632+ if (x==enc_EXCEPTION)
3633+ return -1;
3634+ else if (x==enc_FAILED) {
3635+ raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3636+ return -1;
3637+ }
3638+ }
3639+ }
3640+ *inpos = collendpos;
3641+ break;
3642+ default:
3643+ repunicode = unicode_encode_call_errorhandler(errors, errorHandler,
3644+ encoding, reason, p, size, exceptionObject,
3645+ collstartpos, collendpos, &newpos);
3646+ if (repunicode == NULL)
3647+ return -1;
3648+ /* generate replacement */
3649+ repsize = PyUnicode_GET_SIZE(repunicode);
3650+ for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {
3651+ x = charmapencode_output(*uni2, mapping, res, respos);
3652+ if (x==enc_EXCEPTION) {
3653+ return -1;
3654+ }
3655+ else if (x==enc_FAILED) {
3656+ Py_DECREF(repunicode);
3657+ raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3658+ return -1;
3659+ }
3660+ }
3661+ *inpos = newpos;
3662+ Py_DECREF(repunicode);
3663+ }
3664+ return 0;
3665+}
3666+
3667+PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p,
3668+ Py_ssize_t size,
3669+ PyObject *mapping,
3670+ const char *errors)
3671+{
3672+ /* output object */
3673+ PyObject *res = NULL;
3674+ /* current input position */
3675+ Py_ssize_t inpos = 0;
3676+ /* current output position */
3677+ Py_ssize_t respos = 0;
3678+ PyObject *errorHandler = NULL;
3679+ PyObject *exc = NULL;
3680+ /* the following variable is used for caching string comparisons
3681+ * -1=not initialized, 0=unknown, 1=strict, 2=replace,
3682+ * 3=ignore, 4=xmlcharrefreplace */
3683+ int known_errorHandler = -1;
3684+
3685+ /* Default to Latin-1 */
3686+ if (mapping == NULL)
3687+ return PyUnicode_EncodeLatin1(p, size, errors);
3688+
3689+ /* allocate enough for a simple encoding without
3690+ replacements, if we need more, we'll resize */
3691+ res = PyString_FromStringAndSize(NULL, size);
3692+ if (res == NULL)
3693+ goto onError;
3694+ if (size == 0)
3695+ return res;
3696+
3697+ while (inpos<size) {
3698+ /* try to encode it */
3699+ charmapencode_result x = charmapencode_output(p[inpos], mapping, &res, &respos);
3700+ if (x==enc_EXCEPTION) /* error */
3701+ goto onError;
3702+ if (x==enc_FAILED) { /* unencodable character */
3703+ if (charmap_encoding_error(p, size, &inpos, mapping,
3704+ &exc,
3705+ &known_errorHandler, &errorHandler, errors,
3706+ &res, &respos)) {
3707+ goto onError;
3708+ }
3709+ }
3710+ else
3711+ /* done with this character => adjust input position */
3712+ ++inpos;
3713+ }
3714+
3715+ /* Resize if we allocated to much */
3716+ if (respos<PyString_GET_SIZE(res)) {
3717+ if (_PyString_Resize(&res, respos))
3718+ goto onError;
3719+ }
3720+ Py_XDECREF(exc);
3721+ Py_XDECREF(errorHandler);
3722+ return res;
3723+
3724+ onError:
3725+ Py_XDECREF(res);
3726+ Py_XDECREF(exc);
3727+ Py_XDECREF(errorHandler);
3728+ return NULL;
3729+}
3730+
3731+PyObject *PyUnicode_AsCharmapString(PyObject *unicode,
3732+ PyObject *mapping)
3733+{
3734+ if (!PyUnicode_Check(unicode) || mapping == NULL) {
3735+ PyErr_BadArgument();
3736+ return NULL;
3737+ }
3738+ return PyUnicode_EncodeCharmap(PyUnicode_AS_UNICODE(unicode),
3739+ PyUnicode_GET_SIZE(unicode),
3740+ mapping,
3741+ NULL);
3742+}
3743+
3744+/* create or adjust a UnicodeTranslateError */
3745+static void make_translate_exception(PyObject **exceptionObject,
3746+ const Py_UNICODE *unicode, Py_ssize_t size,
3747+ Py_ssize_t startpos, Py_ssize_t endpos,
3748+ const char *reason)
3749+{
3750+ if (*exceptionObject == NULL) {
3751+ *exceptionObject = PyUnicodeTranslateError_Create(
3752+ unicode, size, startpos, endpos, reason);
3753+ }
3754+ else {
3755+ if (PyUnicodeTranslateError_SetStart(*exceptionObject, startpos))
3756+ goto onError;
3757+ if (PyUnicodeTranslateError_SetEnd(*exceptionObject, endpos))
3758+ goto onError;
3759+ if (PyUnicodeTranslateError_SetReason(*exceptionObject, reason))
3760+ goto onError;
3761+ return;
3762+ onError:
3763+ Py_DECREF(*exceptionObject);
3764+ *exceptionObject = NULL;
3765+ }
3766+}
3767+
3768+/* raises a UnicodeTranslateError */
3769+static void raise_translate_exception(PyObject **exceptionObject,
3770+ const Py_UNICODE *unicode, Py_ssize_t size,
3771+ Py_ssize_t startpos, Py_ssize_t endpos,
3772+ const char *reason)
3773+{
3774+ make_translate_exception(exceptionObject,
3775+ unicode, size, startpos, endpos, reason);
3776+ if (*exceptionObject != NULL)
3777+ PyCodec_StrictErrors(*exceptionObject);
3778+}
3779+
3780+/* error handling callback helper:
3781+ build arguments, call the callback and check the arguments,
3782+ put the result into newpos and return the replacement string, which
3783+ has to be freed by the caller */
3784+static PyObject *unicode_translate_call_errorhandler(const char *errors,
3785+ PyObject **errorHandler,
3786+ const char *reason,
3787+ const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
3788+ Py_ssize_t startpos, Py_ssize_t endpos,
3789+ Py_ssize_t *newpos)
3790+{
3791+ static char *argparse = "O!n;translating error handler must return (unicode, int) tuple";
3792+
3793+ Py_ssize_t i_newpos;
3794+ PyObject *restuple;
3795+ PyObject *resunicode;
3796+
3797+ if (*errorHandler == NULL) {
3798+ *errorHandler = PyCodec_LookupError(errors);
3799+ if (*errorHandler == NULL)
3800+ return NULL;
3801+ }
3802+
3803+ make_translate_exception(exceptionObject,
3804+ unicode, size, startpos, endpos, reason);
3805+ if (*exceptionObject == NULL)
3806+ return NULL;
3807+
3808+ restuple = PyObject_CallFunctionObjArgs(
3809+ *errorHandler, *exceptionObject, NULL);
3810+ if (restuple == NULL)
3811+ return NULL;
3812+ if (!PyTuple_Check(restuple)) {
3813+ PyErr_Format(PyExc_TypeError, &argparse[4]);
3814+ Py_DECREF(restuple);
3815+ return NULL;
3816+ }
3817+ if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type,
3818+ &resunicode, &i_newpos)) {
3819+ Py_DECREF(restuple);
3820+ return NULL;
3821+ }
3822+ if (i_newpos<0)
3823+ *newpos = size+i_newpos;
3824+ else
3825+ *newpos = i_newpos;
3826+ if (*newpos<0 || *newpos>size) {
3827+ PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
3828+ Py_DECREF(restuple);
3829+ return NULL;
3830+ }
3831+ Py_INCREF(resunicode);
3832+ Py_DECREF(restuple);
3833+ return resunicode;
3834+}
3835+
3836+/* Lookup the character ch in the mapping and put the result in result,
3837+ which must be decrefed by the caller.
3838+ Return 0 on success, -1 on error */
3839+static
3840+int charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result)
3841+{
3842+ PyObject *w = PyInt_FromLong((long)c);
3843+ PyObject *x;
3844+
3845+ if (w == NULL)
3846+ return -1;
3847+ x = PyObject_GetItem(mapping, w);
3848+ Py_DECREF(w);
3849+ if (x == NULL) {
3850+ if (PyErr_ExceptionMatches(PyExc_LookupError)) {
3851+ /* No mapping found means: use 1:1 mapping. */
3852+ PyErr_Clear();
3853+ *result = NULL;
3854+ return 0;
3855+ } else
3856+ return -1;
3857+ }
3858+ else if (x == Py_None) {
3859+ *result = x;
3860+ return 0;
3861+ }
3862+ else if (PyInt_Check(x)) {
3863+ long value = PyInt_AS_LONG(x);
3864+ long max = PyUnicode_GetMax();
3865+ if (value < 0 || value > max) {
3866+ PyErr_Format(PyExc_TypeError,
3867+ "character mapping must be in range(0x%lx)", max+1);
3868+ Py_DECREF(x);
3869+ return -1;
3870+ }
3871+ *result = x;
3872+ return 0;
3873+ }
3874+ else if (PyUnicode_Check(x)) {
3875+ *result = x;
3876+ return 0;
3877+ }
3878+ else {
3879+ /* wrong return value */
3880+ PyErr_SetString(PyExc_TypeError,
3881+ "character mapping must return integer, None or unicode");
3882+ Py_DECREF(x);
3883+ return -1;
3884+ }
3885+}
3886+/* ensure that *outobj is at least requiredsize characters long,
3887+if not reallocate and adjust various state variables.
3888+Return 0 on success, -1 on error */
3889+static
3890+int charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp,
3891+ Py_ssize_t requiredsize)
3892+{
3893+ Py_ssize_t oldsize = PyUnicode_GET_SIZE(*outobj);
3894+ if (requiredsize > oldsize) {
3895+ /* remember old output position */
3896+ Py_ssize_t outpos = *outp-PyUnicode_AS_UNICODE(*outobj);
3897+ /* exponentially overallocate to minimize reallocations */
3898+ if (requiredsize < 2 * oldsize)
3899+ requiredsize = 2 * oldsize;
3900+ if (_PyUnicode_Resize(outobj, requiredsize) < 0)
3901+ return -1;
3902+ *outp = PyUnicode_AS_UNICODE(*outobj) + outpos;
3903+ }
3904+ return 0;
3905+}
3906+/* lookup the character, put the result in the output string and adjust
3907+ various state variables. Return a new reference to the object that
3908+ was put in the output buffer in *result, or Py_None, if the mapping was
3909+ undefined (in which case no character was written).
3910+ The called must decref result.
3911+ Return 0 on success, -1 on error. */
3912+static
3913+int charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp,
3914+ Py_ssize_t insize, PyObject *mapping, PyObject **outobj, Py_UNICODE **outp,
3915+ PyObject **res)
3916+{
3917+ if (charmaptranslate_lookup(*curinp, mapping, res))
3918+ return -1;
3919+ if (*res==NULL) {
3920+ /* not found => default to 1:1 mapping */
3921+ *(*outp)++ = *curinp;
3922+ }
3923+ else if (*res==Py_None)
3924+ ;
3925+ else if (PyInt_Check(*res)) {
3926+ /* no overflow check, because we know that the space is enough */
3927+ *(*outp)++ = (Py_UNICODE)PyInt_AS_LONG(*res);
3928+ }
3929+ else if (PyUnicode_Check(*res)) {
3930+ Py_ssize_t repsize = PyUnicode_GET_SIZE(*res);
3931+ if (repsize==1) {
3932+ /* no overflow check, because we know that the space is enough */
3933+ *(*outp)++ = *PyUnicode_AS_UNICODE(*res);
3934+ }
3935+ else if (repsize!=0) {
3936+ /* more than one character */
3937+ Py_ssize_t requiredsize = (*outp-PyUnicode_AS_UNICODE(*outobj)) +
3938+ (insize - (curinp-startinp)) +
3939+ repsize - 1;
3940+ if (charmaptranslate_makespace(outobj, outp, requiredsize))
3941+ return -1;
3942+ memcpy(*outp, PyUnicode_AS_UNICODE(*res), sizeof(Py_UNICODE)*repsize);
3943+ *outp += repsize;
3944+ }
3945+ }
3946+ else
3947+ return -1;
3948+ return 0;
3949+}
3950+
3951+PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p,
3952+ Py_ssize_t size,
3953+ PyObject *mapping,
3954+ const char *errors)
3955+{
3956+ /* output object */
3957+ PyObject *res = NULL;
3958+ /* pointers to the beginning and end+1 of input */
3959+ const Py_UNICODE *startp = p;
3960+ const Py_UNICODE *endp = p + size;
3961+ /* pointer into the output */
3962+ Py_UNICODE *str;
3963+ /* current output position */
3964+ Py_ssize_t respos = 0;
3965+ char *reason = "character maps to <undefined>";
3966+ PyObject *errorHandler = NULL;
3967+ PyObject *exc = NULL;
3968+ /* the following variable is used for caching string comparisons
3969+ * -1=not initialized, 0=unknown, 1=strict, 2=replace,
3970+ * 3=ignore, 4=xmlcharrefreplace */
3971+ int known_errorHandler = -1;
3972+
3973+ if (mapping == NULL) {
3974+ PyErr_BadArgument();
3975+ return NULL;
3976+ }
3977+
3978+ /* allocate enough for a simple 1:1 translation without
3979+ replacements, if we need more, we'll resize */
3980+ res = PyUnicode_FromUnicode(NULL, size);
3981+ if (res == NULL)
3982+ goto onError;
3983+ if (size == 0)
3984+ return res;
3985+ str = PyUnicode_AS_UNICODE(res);
3986+
3987+ while (p<endp) {
3988+ /* try to encode it */
3989+ PyObject *x = NULL;
3990+ if (charmaptranslate_output(startp, p, size, mapping, &res, &str, &x)) {
3991+ Py_XDECREF(x);
3992+ goto onError;
3993+ }
3994+ Py_XDECREF(x);
3995+ if (x!=Py_None) /* it worked => adjust input pointer */
3996+ ++p;
3997+ else { /* untranslatable character */
3998+ PyObject *repunicode = NULL; /* initialize to prevent gcc warning */
3999+ Py_ssize_t repsize;
4000+ Py_ssize_t newpos;
4001+ Py_UNICODE *uni2;
4002+ /* startpos for collecting untranslatable chars */
4003+ const Py_UNICODE *collstart = p;
4004+ const Py_UNICODE *collend = p+1;
4005+ const Py_UNICODE *coll;
4006+
4007+ /* find all untranslatable characters */
4008+ while (collend < endp) {
4009+ if (charmaptranslate_lookup(*collend, mapping, &x))
4010+ goto onError;
4011+ Py_XDECREF(x);
4012+ if (x!=Py_None)
4013+ break;
4014+ ++collend;
4015+ }
4016+ /* cache callback name lookup
4017+ * (if not done yet, i.e. it's the first error) */
4018+ if (known_errorHandler==-1) {
4019+ if ((errors==NULL) || (!strcmp(errors, "strict")))
4020+ known_errorHandler = 1;
4021+ else if (!strcmp(errors, "replace"))
4022+ known_errorHandler = 2;
4023+ else if (!strcmp(errors, "ignore"))
4024+ known_errorHandler = 3;
4025+ else if (!strcmp(errors, "xmlcharrefreplace"))
4026+ known_errorHandler = 4;
4027+ else
4028+ known_errorHandler = 0;
4029+ }
4030+ switch (known_errorHandler) {
4031+ case 1: /* strict */
4032+ raise_translate_exception(&exc, startp, size, collstart-startp, collend-startp, reason);
4033+ goto onError;
4034+ case 2: /* replace */
4035+ /* No need to check for space, this is a 1:1 replacement */
4036+ for (coll = collstart; coll<collend; ++coll)
4037+ *str++ = '?';
4038+ /* fall through */
4039+ case 3: /* ignore */
4040+ p = collend;
4041+ break;
4042+ case 4: /* xmlcharrefreplace */
4043+ /* generate replacement (temporarily (mis)uses p) */
4044+ for (p = collstart; p < collend; ++p) {
4045+ char buffer[2+29+1+1];
4046+ char *cp;
4047+ sprintf(buffer, "&#%d;", (int)*p);
4048+ if (charmaptranslate_makespace(&res, &str,
4049+ (str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend)))
4050+ goto onError;
4051+ for (cp = buffer; *cp; ++cp)
4052+ *str++ = *cp;
4053+ }
4054+ p = collend;
4055+ break;
4056+ default:
4057+ repunicode = unicode_translate_call_errorhandler(errors, &errorHandler,
4058+ reason, startp, size, &exc,
4059+ collstart-startp, collend-startp, &newpos);
4060+ if (repunicode == NULL)
4061+ goto onError;
4062+ /* generate replacement */
4063+ repsize = PyUnicode_GET_SIZE(repunicode);
4064+ if (charmaptranslate_makespace(&res, &str,
4065+ (str-PyUnicode_AS_UNICODE(res))+repsize+(endp-collend))) {
4066+ Py_DECREF(repunicode);
4067+ goto onError;
4068+ }
4069+ for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2)
4070+ *str++ = *uni2;
4071+ p = startp + newpos;
4072+ Py_DECREF(repunicode);
4073+ }
4074+ }
4075+ }
4076+ /* Resize if we allocated to much */
4077+ respos = str-PyUnicode_AS_UNICODE(res);
4078+ if (respos<PyUnicode_GET_SIZE(res)) {
4079+ if (_PyUnicode_Resize(&res, respos) < 0)
4080+ goto onError;
4081+ }
4082+ Py_XDECREF(exc);
4083+ Py_XDECREF(errorHandler);
4084+ return res;
4085+
4086+ onError:
4087+ Py_XDECREF(res);
4088+ Py_XDECREF(exc);
4089+ Py_XDECREF(errorHandler);
4090+ return NULL;
4091+}
4092+
4093+PyObject *PyUnicode_Translate(PyObject *str,
4094+ PyObject *mapping,
4095+ const char *errors)
4096+{
4097+ PyObject *result;
4098+
4099+ str = PyUnicode_FromObject(str);
4100+ if (str == NULL)
4101+ goto onError;
4102+ result = PyUnicode_TranslateCharmap(PyUnicode_AS_UNICODE(str),
4103+ PyUnicode_GET_SIZE(str),
4104+ mapping,
4105+ errors);
4106+ Py_DECREF(str);
4107+ return result;
4108+
4109+ onError:
4110+ Py_XDECREF(str);
4111+ return NULL;
4112+}
4113+
4114+/* --- Decimal Encoder ---------------------------------------------------- */
4115+
4116+int PyUnicode_EncodeDecimal(Py_UNICODE *s,
4117+ Py_ssize_t length,
4118+ char *output,
4119+ const char *errors)
4120+{
4121+ Py_UNICODE *p, *end;
4122+ PyObject *errorHandler = NULL;
4123+ PyObject *exc = NULL;
4124+ const char *encoding = "decimal";
4125+ const char *reason = "invalid decimal Unicode string";
4126+ /* the following variable is used for caching string comparisons
4127+ * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
4128+ int known_errorHandler = -1;
4129+
4130+ if (output == NULL) {
4131+ PyErr_BadArgument();
4132+ return -1;
4133+ }
4134+
4135+ p = s;
4136+ end = s + length;
4137+ while (p < end) {
4138+ register Py_UNICODE ch = *p;
4139+ int decimal;
4140+ PyObject *repunicode;
4141+ Py_ssize_t repsize;
4142+ Py_ssize_t newpos;
4143+ Py_UNICODE *uni2;
4144+ Py_UNICODE *collstart;
4145+ Py_UNICODE *collend;
4146+
4147+ if (Py_UNICODE_ISSPACE(ch)) {
4148+ *output++ = ' ';
4149+ ++p;
4150+ continue;
4151+ }
4152+ decimal = Py_UNICODE_TODECIMAL(ch);
4153+ if (decimal >= 0) {
4154+ *output++ = '0' + decimal;
4155+ ++p;
4156+ continue;
4157+ }
4158+ if (0 < ch && ch < 256) {
4159+ *output++ = (char)ch;
4160+ ++p;
4161+ continue;
4162+ }
4163+ /* All other characters are considered unencodable */
4164+ collstart = p;
4165+ collend = p+1;
4166+ while (collend < end) {
4167+ if ((0 < *collend && *collend < 256) ||
4168+ !Py_UNICODE_ISSPACE(*collend) ||
4169+ Py_UNICODE_TODECIMAL(*collend))
4170+ break;
4171+ }
4172+ /* cache callback name lookup
4173+ * (if not done yet, i.e. it's the first error) */
4174+ if (known_errorHandler==-1) {
4175+ if ((errors==NULL) || (!strcmp(errors, "strict")))
4176+ known_errorHandler = 1;
4177+ else if (!strcmp(errors, "replace"))
4178+ known_errorHandler = 2;
4179+ else if (!strcmp(errors, "ignore"))
4180+ known_errorHandler = 3;
4181+ else if (!strcmp(errors, "xmlcharrefreplace"))
4182+ known_errorHandler = 4;
4183+ else
4184+ known_errorHandler = 0;
4185+ }
4186+ switch (known_errorHandler) {
4187+ case 1: /* strict */
4188+ raise_encode_exception(&exc, encoding, s, length, collstart-s, collend-s, reason);
4189+ goto onError;
4190+ case 2: /* replace */
4191+ for (p = collstart; p < collend; ++p)
4192+ *output++ = '?';
4193+ /* fall through */
4194+ case 3: /* ignore */
4195+ p = collend;
4196+ break;
4197+ case 4: /* xmlcharrefreplace */
4198+ /* generate replacement (temporarily (mis)uses p) */
4199+ for (p = collstart; p < collend; ++p)
4200+ output += sprintf(output, "&#%d;", (int)*p);
4201+ p = collend;
4202+ break;
4203+ default:
4204+ repunicode = unicode_encode_call_errorhandler(errors, &errorHandler,
4205+ encoding, reason, s, length, &exc,
4206+ collstart-s, collend-s, &newpos);
4207+ if (repunicode == NULL)
4208+ goto onError;
4209+ /* generate replacement */
4210+ repsize = PyUnicode_GET_SIZE(repunicode);
4211+ for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {
4212+ Py_UNICODE ch = *uni2;
4213+ if (Py_UNICODE_ISSPACE(ch))
4214+ *output++ = ' ';
4215+ else {
4216+ decimal = Py_UNICODE_TODECIMAL(ch);
4217+ if (decimal >= 0)
4218+ *output++ = '0' + decimal;
4219+ else if (0 < ch && ch < 256)
4220+ *output++ = (char)ch;
4221+ else {
4222+ Py_DECREF(repunicode);
4223+ raise_encode_exception(&exc, encoding,
4224+ s, length, collstart-s, collend-s, reason);
4225+ goto onError;
4226+ }
4227+ }
4228+ }
4229+ p = s + newpos;
4230+ Py_DECREF(repunicode);
4231+ }
4232+ }
4233+ /* 0-terminate the output string */
4234+ *output++ = '\0';
4235+ Py_XDECREF(exc);
4236+ Py_XDECREF(errorHandler);
4237+ return 0;
4238+
4239+ onError:
4240+ Py_XDECREF(exc);
4241+ Py_XDECREF(errorHandler);
4242+ return -1;
4243+}
4244+
4245+/* --- Helpers ------------------------------------------------------------ */
4246+
4247+#define STRINGLIB_CHAR Py_UNICODE
4248+
4249+#define STRINGLIB_LEN PyUnicode_GET_SIZE
4250+#define STRINGLIB_NEW PyUnicode_FromUnicode
4251+#define STRINGLIB_STR PyUnicode_AS_UNICODE
4252+
4253+Py_LOCAL_INLINE(int)
4254+STRINGLIB_CMP(const Py_UNICODE* str, const Py_UNICODE* other, Py_ssize_t len)
4255+{
4256+ if (str[0] != other[0])
4257+ return 1;
4258+ return memcmp((void*) str, (void*) other, len * sizeof(Py_UNICODE));
4259+}
4260+
4261+#define STRINGLIB_EMPTY unicode_empty
4262+
4263+#include "stringlib/fastsearch.h"
4264+
4265+#include "stringlib/count.h"
4266+#include "stringlib/find.h"
4267+#include "stringlib/partition.h"
4268+
4269+/* helper macro to fixup start/end slice values */
4270+#define FIX_START_END(obj) \
4271+ if (start < 0) \
4272+ start += (obj)->length; \
4273+ if (start < 0) \
4274+ start = 0; \
4275+ if (end > (obj)->length) \
4276+ end = (obj)->length; \
4277+ if (end < 0) \
4278+ end += (obj)->length; \
4279+ if (end < 0) \
4280+ end = 0;
4281+
4282+Py_ssize_t PyUnicode_Count(PyObject *str,
4283+ PyObject *substr,
4284+ Py_ssize_t start,
4285+ Py_ssize_t end)
4286+{
4287+ Py_ssize_t result;
4288+ PyUnicodeObject* str_obj;
4289+ PyUnicodeObject* sub_obj;
4290+
4291+ str_obj = (PyUnicodeObject*) PyUnicode_FromObject(str);
4292+ if (!str_obj)
4293+ return -1;
4294+ sub_obj = (PyUnicodeObject*) PyUnicode_FromObject(substr);
4295+ if (!sub_obj) {
4296+ Py_DECREF(str_obj);
4297+ return -1;
4298+ }
4299+
4300+ FIX_START_END(str_obj);
4301+
4302+ result = stringlib_count(
4303+ str_obj->str + start, end - start, sub_obj->str, sub_obj->length
4304+ );
4305+
4306+ Py_DECREF(sub_obj);
4307+ Py_DECREF(str_obj);
4308+
4309+ return result;
4310+}
4311+
4312+Py_ssize_t PyUnicode_Find(PyObject *str,
4313+ PyObject *sub,
4314+ Py_ssize_t start,
4315+ Py_ssize_t end,
4316+ int direction)
4317+{
4318+ Py_ssize_t result;
4319+
4320+ str = PyUnicode_FromObject(str);
4321+ if (!str)
4322+ return -2;
4323+ sub = PyUnicode_FromObject(sub);
4324+ if (!sub) {
4325+ Py_DECREF(str);
4326+ return -2;
4327+ }
4328+
4329+ if (direction > 0)
4330+ result = stringlib_find_slice(
4331+ PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str),
4332+ PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub),
4333+ start, end
4334+ );
4335+ else
4336+ result = stringlib_rfind_slice(
4337+ PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str),
4338+ PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub),
4339+ start, end
4340+ );
4341+
4342+ Py_DECREF(str);
4343+ Py_DECREF(sub);
4344+
4345+ return result;
4346+}
4347+
4348+static
4349+int tailmatch(PyUnicodeObject *self,
4350+ PyUnicodeObject *substring,
4351+ Py_ssize_t start,
4352+ Py_ssize_t end,
4353+ int direction)
4354+{
4355+ if (substring->length == 0)
4356+ return 1;
4357+
4358+ FIX_START_END(self);
4359+
4360+ end -= substring->length;
4361+ if (end < start)
4362+ return 0;
4363+
4364+ if (direction > 0) {
4365+ if (Py_UNICODE_MATCH(self, end, substring))
4366+ return 1;
4367+ } else {
4368+ if (Py_UNICODE_MATCH(self, start, substring))
4369+ return 1;
4370+ }
4371+
4372+ return 0;
4373+}
4374+
4375+Py_ssize_t PyUnicode_Tailmatch(PyObject *str,
4376+ PyObject *substr,
4377+ Py_ssize_t start,
4378+ Py_ssize_t end,
4379+ int direction)
4380+{
4381+ Py_ssize_t result;
4382+
4383+ str = PyUnicode_FromObject(str);
4384+ if (str == NULL)
4385+ return -1;
4386+ substr = PyUnicode_FromObject(substr);
4387+ if (substr == NULL) {
4388+ Py_DECREF(str);
4389+ return -1;
4390+ }
4391+
4392+ result = tailmatch((PyUnicodeObject *)str,
4393+ (PyUnicodeObject *)substr,
4394+ start, end, direction);
4395+ Py_DECREF(str);
4396+ Py_DECREF(substr);
4397+ return result;
4398+}
4399+
4400+/* Apply fixfct filter to the Unicode object self and return a
4401+ reference to the modified object */
4402+
4403+static
4404+PyObject *fixup(PyUnicodeObject *self,
4405+ int (*fixfct)(PyUnicodeObject *s))
4406+{
4407+
4408+ PyUnicodeObject *u;
4409+
4410+ u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length);
4411+ if (u == NULL)
4412+ return NULL;
4413+
4414+ Py_UNICODE_COPY(u->str, self->str, self->length);
4415+
4416+ if (!fixfct(u) && PyUnicode_CheckExact(self)) {
4417+ /* fixfct should return TRUE if it modified the buffer. If
4418+ FALSE, return a reference to the original buffer instead
4419+ (to save space, not time) */
4420+ Py_INCREF(self);
4421+ Py_DECREF(u);
4422+ return (PyObject*) self;
4423+ }
4424+ return (PyObject*) u;
4425+}
4426+
4427+static
4428+int fixupper(PyUnicodeObject *self)
4429+{
4430+ Py_ssize_t len = self->length;
4431+ Py_UNICODE *s = self->str;
4432+ int status = 0;
4433+
4434+ while (len-- > 0) {
4435+ register Py_UNICODE ch;
4436+
4437+ ch = Py_UNICODE_TOUPPER(*s);
4438+ if (ch != *s) {
4439+ status = 1;
4440+ *s = ch;
4441+ }
4442+ s++;
4443+ }
4444+
4445+ return status;
4446+}
4447+
4448+static
4449+int fixlower(PyUnicodeObject *self)
4450+{
4451+ Py_ssize_t len = self->length;
4452+ Py_UNICODE *s = self->str;
4453+ int status = 0;
4454+
4455+ while (len-- > 0) {
4456+ register Py_UNICODE ch;
4457+
4458+ ch = Py_UNICODE_TOLOWER(*s);
4459+ if (ch != *s) {
4460+ status = 1;
4461+ *s = ch;
4462+ }
4463+ s++;
4464+ }
4465+
4466+ return status;
4467+}
4468+
4469+static
4470+int fixswapcase(PyUnicodeObject *self)
4471+{
4472+ Py_ssize_t len = self->length;
4473+ Py_UNICODE *s = self->str;
4474+ int status = 0;
4475+
4476+ while (len-- > 0) {
4477+ if (Py_UNICODE_ISUPPER(*s)) {
4478+ *s = Py_UNICODE_TOLOWER(*s);
4479+ status = 1;
4480+ } else if (Py_UNICODE_ISLOWER(*s)) {
4481+ *s = Py_UNICODE_TOUPPER(*s);
4482+ status = 1;
4483+ }
4484+ s++;
4485+ }
4486+
4487+ return status;
4488+}
4489+
4490+static
4491+int fixcapitalize(PyUnicodeObject *self)
4492+{
4493+ Py_ssize_t len = self->length;
4494+ Py_UNICODE *s = self->str;
4495+ int status = 0;
4496+
4497+ if (len == 0)
4498+ return 0;
4499+ if (Py_UNICODE_ISLOWER(*s)) {
4500+ *s = Py_UNICODE_TOUPPER(*s);
4501+ status = 1;
4502+ }
4503+ s++;
4504+ while (--len > 0) {
4505+ if (Py_UNICODE_ISUPPER(*s)) {
4506+ *s = Py_UNICODE_TOLOWER(*s);
4507+ status = 1;
4508+ }
4509+ s++;
4510+ }
4511+ return status;
4512+}
4513+
4514+static
4515+int fixtitle(PyUnicodeObject *self)
4516+{
4517+ register Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
4518+ register Py_UNICODE *e;
4519+ int previous_is_cased;
4520+
4521+ /* Shortcut for single character strings */
4522+ if (PyUnicode_GET_SIZE(self) == 1) {
4523+ Py_UNICODE ch = Py_UNICODE_TOTITLE(*p);
4524+ if (*p != ch) {
4525+ *p = ch;
4526+ return 1;
4527+ }
4528+ else
4529+ return 0;
4530+ }
4531+
4532+ e = p + PyUnicode_GET_SIZE(self);
4533+ previous_is_cased = 0;
4534+ for (; p < e; p++) {
4535+ register const Py_UNICODE ch = *p;
4536+
4537+ if (previous_is_cased)
4538+ *p = Py_UNICODE_TOLOWER(ch);
4539+ else
4540+ *p = Py_UNICODE_TOTITLE(ch);
4541+
4542+ if (Py_UNICODE_ISLOWER(ch) ||
4543+ Py_UNICODE_ISUPPER(ch) ||
4544+ Py_UNICODE_ISTITLE(ch))
4545+ previous_is_cased = 1;
4546+ else
4547+ previous_is_cased = 0;
4548+ }
4549+ return 1;
4550+}
4551+
4552+PyObject *
4553+PyUnicode_Join(PyObject *separator, PyObject *seq)
4554+{
4555+ PyObject *internal_separator = NULL;
4556+ const Py_UNICODE blank = ' ';
4557+ const Py_UNICODE *sep = &blank;
4558+ Py_ssize_t seplen = 1;
4559+ PyUnicodeObject *res = NULL; /* the result */
4560+ Py_ssize_t res_alloc = 100; /* # allocated bytes for string in res */
4561+ Py_ssize_t res_used; /* # used bytes */
4562+ Py_UNICODE *res_p; /* pointer to free byte in res's string area */
4563+ PyObject *fseq; /* PySequence_Fast(seq) */
4564+ Py_ssize_t seqlen; /* len(fseq) -- number of items in sequence */
4565+ PyObject *item;
4566+ Py_ssize_t i;
4567+
4568+ fseq = PySequence_Fast(seq, "");
4569+ if (fseq == NULL) {
4570+ return NULL;
4571+ }
4572+
4573+ /* Grrrr. A codec may be invoked to convert str objects to
4574+ * Unicode, and so it's possible to call back into Python code
4575+ * during PyUnicode_FromObject(), and so it's possible for a sick
4576+ * codec to change the size of fseq (if seq is a list). Therefore
4577+ * we have to keep refetching the size -- can't assume seqlen
4578+ * is invariant.
4579+ */
4580+ seqlen = PySequence_Fast_GET_SIZE(fseq);
4581+ /* If empty sequence, return u"". */
4582+ if (seqlen == 0) {
4583+ res = _PyUnicode_New(0); /* empty sequence; return u"" */
4584+ goto Done;
4585+ }
4586+ /* If singleton sequence with an exact Unicode, return that. */
4587+ if (seqlen == 1) {
4588+ item = PySequence_Fast_GET_ITEM(fseq, 0);
4589+ if (PyUnicode_CheckExact(item)) {
4590+ Py_INCREF(item);
4591+ res = (PyUnicodeObject *)item;
4592+ goto Done;
4593+ }
4594+ }
4595+
4596+ /* At least two items to join, or one that isn't exact Unicode. */
4597+ if (seqlen > 1) {
4598+ /* Set up sep and seplen -- they're needed. */
4599+ if (separator == NULL) {
4600+ sep = &blank;
4601+ seplen = 1;
4602+ }
4603+ else {
4604+ internal_separator = PyUnicode_FromObject(separator);
4605+ if (internal_separator == NULL)
4606+ goto onError;
4607+ sep = PyUnicode_AS_UNICODE(internal_separator);
4608+ seplen = PyUnicode_GET_SIZE(internal_separator);
4609+ /* In case PyUnicode_FromObject() mutated seq. */
4610+ seqlen = PySequence_Fast_GET_SIZE(fseq);
4611+ }
4612+ }
4613+
4614+ /* Get space. */
4615+ res = _PyUnicode_New(res_alloc);
4616+ if (res == NULL)
4617+ goto onError;
4618+ res_p = PyUnicode_AS_UNICODE(res);
4619+ res_used = 0;
4620+
4621+ for (i = 0; i < seqlen; ++i) {
4622+ Py_ssize_t itemlen;
4623+ Py_ssize_t new_res_used;
4624+
4625+ item = PySequence_Fast_GET_ITEM(fseq, i);
4626+ /* Convert item to Unicode. */
4627+ if (! PyUnicode_Check(item) && ! PyString_Check(item)) {
4628+ PyErr_Format(PyExc_TypeError,
4629+ "sequence item %zd: expected string or Unicode,"
4630+ " %.80s found",
4631+ i, item->ob_type->tp_name);
4632+ goto onError;
4633+ }
4634+ item = PyUnicode_FromObject(item);
4635+ if (item == NULL)
4636+ goto onError;
4637+ /* We own a reference to item from here on. */
4638+
4639+ /* In case PyUnicode_FromObject() mutated seq. */
4640+ seqlen = PySequence_Fast_GET_SIZE(fseq);
4641+
4642+ /* Make sure we have enough space for the separator and the item. */
4643+ itemlen = PyUnicode_GET_SIZE(item);
4644+ new_res_used = res_used + itemlen;
4645+ if (new_res_used < 0)
4646+ goto Overflow;
4647+ if (i < seqlen - 1) {
4648+ new_res_used += seplen;
4649+ if (new_res_used < 0)
4650+ goto Overflow;
4651+ }
4652+ if (new_res_used > res_alloc) {
4653+ /* double allocated size until it's big enough */
4654+ do {
4655+ res_alloc += res_alloc;
4656+ if (res_alloc <= 0)
4657+ goto Overflow;
4658+ } while (new_res_used > res_alloc);
4659+ if (_PyUnicode_Resize(&res, res_alloc) < 0) {
4660+ Py_DECREF(item);
4661+ goto onError;
4662+ }
4663+ res_p = PyUnicode_AS_UNICODE(res) + res_used;
4664+ }
4665+
4666+ /* Copy item, and maybe the separator. */
4667+ Py_UNICODE_COPY(res_p, PyUnicode_AS_UNICODE(item), itemlen);
4668+ res_p += itemlen;
4669+ if (i < seqlen - 1) {
4670+ Py_UNICODE_COPY(res_p, sep, seplen);
4671+ res_p += seplen;
4672+ }
4673+ Py_DECREF(item);
4674+ res_used = new_res_used;
4675+ }
4676+
4677+ /* Shrink res to match the used area; this probably can't fail,
4678+ * but it's cheap to check.
4679+ */
4680+ if (_PyUnicode_Resize(&res, res_used) < 0)
4681+ goto onError;
4682+
4683+ Done:
4684+ Py_XDECREF(internal_separator);
4685+ Py_DECREF(fseq);
4686+ return (PyObject *)res;
4687+
4688+ Overflow:
4689+ PyErr_SetString(PyExc_OverflowError,
4690+ "join() result is too long for a Python string");
4691+ Py_DECREF(item);
4692+ /* fall through */
4693+
4694+ onError:
4695+ Py_XDECREF(internal_separator);
4696+ Py_DECREF(fseq);
4697+ Py_XDECREF(res);
4698+ return NULL;
4699+}
4700+
4701+static
4702+PyUnicodeObject *pad(PyUnicodeObject *self,
4703+ Py_ssize_t left,
4704+ Py_ssize_t right,
4705+ Py_UNICODE fill)
4706+{
4707+ PyUnicodeObject *u;
4708+
4709+ if (left < 0)
4710+ left = 0;
4711+ if (right < 0)
4712+ right = 0;
4713+
4714+ if (left == 0 && right == 0 && PyUnicode_CheckExact(self)) {
4715+ Py_INCREF(self);
4716+ return self;
4717+ }
4718+
4719+ u = _PyUnicode_New(left + self->length + right);
4720+ if (u) {
4721+ if (left)
4722+ Py_UNICODE_FILL(u->str, fill, left);
4723+ Py_UNICODE_COPY(u->str + left, self->str, self->length);
4724+ if (right)
4725+ Py_UNICODE_FILL(u->str + left + self->length, fill, right);
4726+ }
4727+
4728+ return u;
4729+}
4730+
4731+#define SPLIT_APPEND(data, left, right) \
4732+ str = PyUnicode_FromUnicode((data) + (left), (right) - (left)); \
4733+ if (!str) \
4734+ goto onError; \
4735+ if (PyList_Append(list, str)) { \
4736+ Py_DECREF(str); \
4737+ goto onError; \
4738+ } \
4739+ else \
4740+ Py_DECREF(str);
4741+
4742+static
4743+PyObject *split_whitespace(PyUnicodeObject *self,
4744+ PyObject *list,
4745+ Py_ssize_t maxcount)
4746+{
4747+ register Py_ssize_t i;
4748+ register Py_ssize_t j;
4749+ Py_ssize_t len = self->length;
4750+ PyObject *str;
4751+
4752+ for (i = j = 0; i < len; ) {
4753+ /* find a token */
4754+ while (i < len && Py_UNICODE_ISSPACE(self->str[i]))
4755+ i++;
4756+ j = i;
4757+ while (i < len && !Py_UNICODE_ISSPACE(self->str[i]))
4758+ i++;
4759+ if (j < i) {
4760+ if (maxcount-- <= 0)
4761+ break;
4762+ SPLIT_APPEND(self->str, j, i);
4763+ while (i < len && Py_UNICODE_ISSPACE(self->str[i]))
4764+ i++;
4765+ j = i;
4766+ }
4767+ }
4768+ if (j < len) {
4769+ SPLIT_APPEND(self->str, j, len);
4770+ }
4771+ return list;
4772+
4773+ onError:
4774+ Py_DECREF(list);
4775+ return NULL;
4776+}
4777+
4778+PyObject *PyUnicode_Splitlines(PyObject *string,
4779+ int keepends)
4780+{
4781+ register Py_ssize_t i;
4782+ register Py_ssize_t j;
4783+ Py_ssize_t len;
4784+ PyObject *list;
4785+ PyObject *str;
4786+ Py_UNICODE *data;
4787+
4788+ string = PyUnicode_FromObject(string);
4789+ if (string == NULL)
4790+ return NULL;
4791+ data = PyUnicode_AS_UNICODE(string);
4792+ len = PyUnicode_GET_SIZE(string);
4793+
4794+ list = PyList_New(0);
4795+ if (!list)
4796+ goto onError;
4797+
4798+ for (i = j = 0; i < len; ) {
4799+ Py_ssize_t eol;
4800+
4801+ /* Find a line and append it */
4802+ while (i < len && !BLOOM_LINEBREAK(data[i]))
4803+ i++;
4804+
4805+ /* Skip the line break reading CRLF as one line break */
4806+ eol = i;
4807+ if (i < len) {
4808+ if (data[i] == '\r' && i + 1 < len &&
4809+ data[i+1] == '\n')
4810+ i += 2;
4811+ else
4812+ i++;
4813+ if (keepends)
4814+ eol = i;
4815+ }
4816+ SPLIT_APPEND(data, j, eol);
4817+ j = i;
4818+ }
4819+ if (j < len) {
4820+ SPLIT_APPEND(data, j, len);
4821+ }
4822+
4823+ Py_DECREF(string);
4824+ return list;
4825+
4826+ onError:
4827+ Py_XDECREF(list);
4828+ Py_DECREF(string);
4829+ return NULL;
4830+}
4831+
4832+static
4833+PyObject *split_char(PyUnicodeObject *self,
4834+ PyObject *list,
4835+ Py_UNICODE ch,
4836+ Py_ssize_t maxcount)
4837+{
4838+ register Py_ssize_t i;
4839+ register Py_ssize_t j;
4840+ Py_ssize_t len = self->length;
4841+ PyObject *str;
4842+
4843+ for (i = j = 0; i < len; ) {
4844+ if (self->str[i] == ch) {
4845+ if (maxcount-- <= 0)
4846+ break;
4847+ SPLIT_APPEND(self->str, j, i);
4848+ i = j = i + 1;
4849+ } else
4850+ i++;
4851+ }
4852+ if (j <= len) {
4853+ SPLIT_APPEND(self->str, j, len);
4854+ }
4855+ return list;
4856+
4857+ onError:
4858+ Py_DECREF(list);
4859+ return NULL;
4860+}
4861+
4862+static
4863+PyObject *split_substring(PyUnicodeObject *self,
4864+ PyObject *list,
4865+ PyUnicodeObject *substring,
4866+ Py_ssize_t maxcount)
4867+{
4868+ register Py_ssize_t i;
4869+ register Py_ssize_t j;
4870+ Py_ssize_t len = self->length;
4871+ Py_ssize_t sublen = substring->length;
4872+ PyObject *str;
4873+
4874+ for (i = j = 0; i <= len - sublen; ) {
4875+ if (Py_UNICODE_MATCH(self, i, substring)) {
4876+ if (maxcount-- <= 0)
4877+ break;
4878+ SPLIT_APPEND(self->str, j, i);
4879+ i = j = i + sublen;
4880+ } else
4881+ i++;
4882+ }
4883+ if (j <= len) {
4884+ SPLIT_APPEND(self->str, j, len);
4885+ }
4886+ return list;
4887+
4888+ onError:
4889+ Py_DECREF(list);
4890+ return NULL;
4891+}
4892+
4893+static
4894+PyObject *rsplit_whitespace(PyUnicodeObject *self,
4895+ PyObject *list,
4896+ Py_ssize_t maxcount)
4897+{
4898+ register Py_ssize_t i;
4899+ register Py_ssize_t j;
4900+ Py_ssize_t len = self->length;
4901+ PyObject *str;
4902+
4903+ for (i = j = len - 1; i >= 0; ) {
4904+ /* find a token */
4905+ while (i >= 0 && Py_UNICODE_ISSPACE(self->str[i]))
4906+ i--;
4907+ j = i;
4908+ while (i >= 0 && !Py_UNICODE_ISSPACE(self->str[i]))
4909+ i--;
4910+ if (j > i) {
4911+ if (maxcount-- <= 0)
4912+ break;
4913+ SPLIT_APPEND(self->str, i + 1, j + 1);
4914+ while (i >= 0 && Py_UNICODE_ISSPACE(self->str[i]))
4915+ i--;
4916+ j = i;
4917+ }
4918+ }
4919+ if (j >= 0) {
4920+ SPLIT_APPEND(self->str, 0, j + 1);
4921+ }
4922+ if (PyList_Reverse(list) < 0)
4923+ goto onError;
4924+ return list;
4925+
4926+ onError:
4927+ Py_DECREF(list);
4928+ return NULL;
4929+}
4930+
4931+static
4932+PyObject *rsplit_char(PyUnicodeObject *self,
4933+ PyObject *list,
4934+ Py_UNICODE ch,
4935+ Py_ssize_t maxcount)
4936+{
4937+ register Py_ssize_t i;
4938+ register Py_ssize_t j;
4939+ Py_ssize_t len = self->length;
4940+ PyObject *str;
4941+
4942+ for (i = j = len - 1; i >= 0; ) {
4943+ if (self->str[i] == ch) {
4944+ if (maxcount-- <= 0)
4945+ break;
4946+ SPLIT_APPEND(self->str, i + 1, j + 1);
4947+ j = i = i - 1;
4948+ } else
4949+ i--;
4950+ }
4951+ if (j >= -1) {
4952+ SPLIT_APPEND(self->str, 0, j + 1);
4953+ }
4954+ if (PyList_Reverse(list) < 0)
4955+ goto onError;
4956+ return list;
4957+
4958+ onError:
4959+ Py_DECREF(list);
4960+ return NULL;
4961+}
4962+
4963+static
4964+PyObject *rsplit_substring(PyUnicodeObject *self,
4965+ PyObject *list,
4966+ PyUnicodeObject *substring,
4967+ Py_ssize_t maxcount)
4968+{
4969+ register Py_ssize_t i;
4970+ register Py_ssize_t j;
4971+ Py_ssize_t len = self->length;
4972+ Py_ssize_t sublen = substring->length;
4973+ PyObject *str;
4974+
4975+ for (i = len - sublen, j = len; i >= 0; ) {
4976+ if (Py_UNICODE_MATCH(self, i, substring)) {
4977+ if (maxcount-- <= 0)
4978+ break;
4979+ SPLIT_APPEND(self->str, i + sublen, j);
4980+ j = i;
4981+ i -= sublen;
4982+ } else
4983+ i--;
4984+ }
4985+ if (j >= 0) {
4986+ SPLIT_APPEND(self->str, 0, j);
4987+ }
4988+ if (PyList_Reverse(list) < 0)
4989+ goto onError;
4990+ return list;
4991+
4992+ onError:
4993+ Py_DECREF(list);
4994+ return NULL;
4995+}
4996+
4997+#undef SPLIT_APPEND
4998+
4999+static

3068 more lines aren’t shown here.