History

add hg and python

cinap_lenrek committed 15 years ago

Commit 4bb9e221 parent 987265c3706 files · +1228772 −-233Browse files

Showing the changes to sys/src/cmd/python/Objects/unicodeobject.c (1 of 3706 files). Show all

Unified · Split

sys/src/cmd/python/Objects/unicodeobject.c+8067−0
@@ -0,0 +1,8067 @@
1/*
2
3Unicode implementation based on original code by Fredrik Lundh,
4modified by Marc-Andre Lemburg <mal@lemburg.com> according to the
5Unicode Integration Proposal (see file Misc/unicode.txt).
6
7Major speed upgrades to the method implementations at the Reykjavik
8NeedForSpeed sprint, by Fredrik Lundh and Andrew Dalke.
9
10Copyright (c) Corporation for National Research Initiatives.
11
12--------------------------------------------------------------------
13The original string type implementation is:
14
15 Copyright (c) 1999 by Secret Labs AB
16 Copyright (c) 1999 by Fredrik Lundh
17
18By obtaining, using, and/or copying this software and/or its
19associated documentation, you agree that you have read, understood,
20and will comply with the following terms and conditions:
21
22Permission to use, copy, modify, and distribute this software and its
23associated documentation for any purpose and without fee is hereby
24granted, provided that the above copyright notice appears in all
25copies, and that both that copyright notice and this permission notice
26appear in supporting documentation, and that the name of Secret Labs
27AB or the author not be used in advertising or publicity pertaining to
28distribution of the software without specific, written prior
29permission.
30
31SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO
32THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
33FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR
34ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
35WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
36ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
37OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
38--------------------------------------------------------------------
39
40*/
41
42#define PY_SSIZE_T_CLEAN
43#include "Python.h"
44
45#include "unicodeobject.h"
46#include "ucnhash.h"
47
48#ifdef MS_WINDOWS
49#include <windows.h>
50#endif
51
52/* Limit for the Unicode object free list */
53
54#define MAX_UNICODE_FREELIST_SIZE 1024
55
56/* Limit for the Unicode object free list stay alive optimization.
57
58 The implementation will keep allocated Unicode memory intact for
59 all objects on the free list having a size less than this
60 limit. This reduces malloc() overhead for small Unicode objects.
61
62 At worst this will result in MAX_UNICODE_FREELIST_SIZE *
63 (sizeof(PyUnicodeObject) + KEEPALIVE_SIZE_LIMIT +
64 malloc()-overhead) bytes of unused garbage.
65
66 Setting the limit to 0 effectively turns the feature off.
67
68 Note: This is an experimental feature ! If you get core dumps when
69 using Unicode objects, turn this feature off.
70
71*/
72
73#define KEEPALIVE_SIZE_LIMIT 9
74
75/* Endianness switches; defaults to little endian */
76
77#ifdef WORDS_BIGENDIAN
78# define BYTEORDER_IS_BIG_ENDIAN
79#else
80# define BYTEORDER_IS_LITTLE_ENDIAN
81#endif
82
83/* --- Globals ------------------------------------------------------------
84
85 The globals are initialized by the _PyUnicode_Init() API and should
86 not be used before calling that API.
87
88*/
89
90
91#ifdef __cplusplus
92extern "C" {
93#endif
94
95/* Free list for Unicode objects */
96static PyUnicodeObject *unicode_freelist;
97static int unicode_freelist_size;
98
99/* The empty Unicode object is shared to improve performance. */
100static PyUnicodeObject *unicode_empty;
101
102/* Single character Unicode strings in the Latin-1 range are being
103 shared as well. */
104static PyUnicodeObject *unicode_latin1[256];
105
106/* Default encoding to use and assume when NULL is passed as encoding
107 parameter; it is initialized by _PyUnicode_Init().
108
109 Always use the PyUnicode_SetDefaultEncoding() and
110 PyUnicode_GetDefaultEncoding() APIs to access this global.
111
112*/
113static char unicode_default_encoding[100];
114
115Py_UNICODE
116PyUnicode_GetMax(void)
117{
118#ifdef Py_UNICODE_WIDE
119 return 0x10FFFF;
120#else
121 /* This is actually an illegal character, so it should
122 not be passed to unichr. */
123 return 0xFFFF;
124#endif
125}
126
127/* --- Bloom Filters ----------------------------------------------------- */
128
129/* stuff to implement simple "bloom filters" for Unicode characters.
130 to keep things simple, we use a single bitmask, using the least 5
131 bits from each unicode characters as the bit index. */
132
133/* the linebreak mask is set up by Unicode_Init below */
134
135#define BLOOM_MASK unsigned long
136
137static BLOOM_MASK bloom_linebreak;
138
139#define BLOOM(mask, ch) ((mask & (1 << ((ch) & 0x1F))))
140
141#define BLOOM_LINEBREAK(ch)\
142 (BLOOM(bloom_linebreak, (ch)) && Py_UNICODE_ISLINEBREAK((ch)))
143
144Py_LOCAL_INLINE(BLOOM_MASK) make_bloom_mask(Py_UNICODE* ptr, Py_ssize_t len)
145{
146 /* calculate simple bloom-style bitmask for a given unicode string */
147
148 long mask;
149 Py_ssize_t i;
150
151 mask = 0;
152 for (i = 0; i < len; i++)
153 mask |= (1 << (ptr[i] & 0x1F));
154
155 return mask;
156}
157
158Py_LOCAL_INLINE(int) unicode_member(Py_UNICODE chr, Py_UNICODE* set, Py_ssize_t setlen)
159{
160 Py_ssize_t i;
161
162 for (i = 0; i < setlen; i++)
163 if (set[i] == chr)
164 return 1;
165
166 return 0;
167}
168
169#define BLOOM_MEMBER(mask, chr, set, setlen)\
170 BLOOM(mask, chr) && unicode_member(chr, set, setlen)
171
172/* --- Unicode Object ----------------------------------------------------- */
173
174static
175int unicode_resize(register PyUnicodeObject *unicode,
176 Py_ssize_t length)
177{
178 void *oldstr;
179
180 /* Shortcut if there's nothing much to do. */
181 if (unicode->length == length)
182 goto reset;
183
184 /* Resizing shared object (unicode_empty or single character
185 objects) in-place is not allowed. Use PyUnicode_Resize()
186 instead ! */
187
188 if (unicode == unicode_empty ||
189 (unicode->length == 1 &&
190 unicode->str[0] < 256U &&
191 unicode_latin1[unicode->str[0]] == unicode)) {
192 PyErr_SetString(PyExc_SystemError,
193 "can't resize shared unicode objects");
194 return -1;
195 }
196
197 /* We allocate one more byte to make sure the string is Ux0000 terminated.
198 The overallocation is also used by fastsearch, which assumes that it's
199 safe to look at str[length] (without making any assumptions about what
200 it contains). */
201
202 oldstr = unicode->str;
203 PyMem_RESIZE(unicode->str, Py_UNICODE, length + 1);
204 if (!unicode->str) {
205 unicode->str = (Py_UNICODE *)oldstr;
206 PyErr_NoMemory();
207 return -1;
208 }
209 unicode->str[length] = 0;
210 unicode->length = length;
211
212 reset:
213 /* Reset the object caches */
214 if (unicode->defenc) {
215 Py_DECREF(unicode->defenc);
216 unicode->defenc = NULL;
217 }
218 unicode->hash = -1;
219
220 return 0;
221}
222
223/* We allocate one more byte to make sure the string is
224 Ux0000 terminated -- XXX is this needed ?
225
226 XXX This allocator could further be enhanced by assuring that the
227 free list never reduces its size below 1.
228
229*/
230
231static
232PyUnicodeObject *_PyUnicode_New(Py_ssize_t length)
233{
234 register PyUnicodeObject *unicode;
235
236 /* Optimization for empty strings */
237 if (length == 0 && unicode_empty != NULL) {
238 Py_INCREF(unicode_empty);
239 return unicode_empty;
240 }
241
242 /* Unicode freelist & memory allocation */
243 if (unicode_freelist) {
244 unicode = unicode_freelist;
245 unicode_freelist = *(PyUnicodeObject **)unicode;
246 unicode_freelist_size--;
247 if (unicode->str) {
248 /* Keep-Alive optimization: we only upsize the buffer,
249 never downsize it. */
250 if ((unicode->length < length) &&
251 unicode_resize(unicode, length) < 0) {
252 PyMem_DEL(unicode->str);
253 goto onError;
254 }
255 }
256 else {
257 unicode->str = PyMem_NEW(Py_UNICODE, length + 1);
258 }
259 PyObject_INIT(unicode, &PyUnicode_Type);
260 }
261 else {
262 unicode = PyObject_New(PyUnicodeObject, &PyUnicode_Type);
263 if (unicode == NULL)
264 return NULL;
265 unicode->str = PyMem_NEW(Py_UNICODE, length + 1);
266 }
267
268 if (!unicode->str) {
269 PyErr_NoMemory();
270 goto onError;
271 }
272 /* Initialize the first element to guard against cases where
273 * the caller fails before initializing str -- unicode_resize()
274 * reads str[0], and the Keep-Alive optimization can keep memory
275 * allocated for str alive across a call to unicode_dealloc(unicode).
276 * We don't want unicode_resize to read uninitialized memory in
277 * that case.
278 */
279 unicode->str[0] = 0;
280 unicode->str[length] = 0;
281 unicode->length = length;
282 unicode->hash = -1;
283 unicode->defenc = NULL;
284 return unicode;
285
286 onError:
287 _Py_ForgetReference((PyObject *)unicode);
288 PyObject_Del(unicode);
289 return NULL;
290}
291
292static
293void unicode_dealloc(register PyUnicodeObject *unicode)
294{
295 if (PyUnicode_CheckExact(unicode) &&
296 unicode_freelist_size < MAX_UNICODE_FREELIST_SIZE) {
297 /* Keep-Alive optimization */
298 if (unicode->length >= KEEPALIVE_SIZE_LIMIT) {
299 PyMem_DEL(unicode->str);
300 unicode->str = NULL;
301 unicode->length = 0;
302 }
303 if (unicode->defenc) {
304 Py_DECREF(unicode->defenc);
305 unicode->defenc = NULL;
306 }
307 /* Add to free list */
308 *(PyUnicodeObject **)unicode = unicode_freelist;
309 unicode_freelist = unicode;
310 unicode_freelist_size++;
311 }
312 else {
313 PyMem_DEL(unicode->str);
314 Py_XDECREF(unicode->defenc);
315 unicode->ob_type->tp_free((PyObject *)unicode);
316 }
317}
318
319int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length)
320{
321 register PyUnicodeObject *v;
322
323 /* Argument checks */
324 if (unicode == NULL) {
325 PyErr_BadInternalCall();
326 return -1;
327 }
328 v = (PyUnicodeObject *)*unicode;
329 if (v == NULL || !PyUnicode_Check(v) || v->ob_refcnt != 1 || length < 0) {
330 PyErr_BadInternalCall();
331 return -1;
332 }
333
334 /* Resizing unicode_empty and single character objects is not
335 possible since these are being shared. We simply return a fresh
336 copy with the same Unicode content. */
337 if (v->length != length &&
338 (v == unicode_empty || v->length == 1)) {
339 PyUnicodeObject *w = _PyUnicode_New(length);
340 if (w == NULL)
341 return -1;
342 Py_UNICODE_COPY(w->str, v->str,
343 length < v->length ? length : v->length);
344 Py_DECREF(*unicode);
345 *unicode = (PyObject *)w;
346 return 0;
347 }
348
349 /* Note that we don't have to modify *unicode for unshared Unicode
350 objects, since we can modify them in-place. */
351 return unicode_resize(v, length);
352}
353
354/* Internal API for use in unicodeobject.c only ! */
355#define _PyUnicode_Resize(unicodevar, length) \
356 PyUnicode_Resize(((PyObject **)(unicodevar)), length)
357
358PyObject *PyUnicode_FromUnicode(const Py_UNICODE *u,
359 Py_ssize_t size)
360{
361 PyUnicodeObject *unicode;
362
363 /* If the Unicode data is known at construction time, we can apply
364 some optimizations which share commonly used objects. */
365 if (u != NULL) {
366
367 /* Optimization for empty strings */
368 if (size == 0 && unicode_empty != NULL) {
369 Py_INCREF(unicode_empty);
370 return (PyObject *)unicode_empty;
371 }
372
373 /* Single character Unicode objects in the Latin-1 range are
374 shared when using this constructor */
375 if (size == 1 && *u < 256) {
376 unicode = unicode_latin1[*u];
377 if (!unicode) {
378 unicode = _PyUnicode_New(1);
379 if (!unicode)
380 return NULL;
381 unicode->str[0] = *u;
382 unicode_latin1[*u] = unicode;
383 }
384 Py_INCREF(unicode);
385 return (PyObject *)unicode;
386 }
387 }
388
389 unicode = _PyUnicode_New(size);
390 if (!unicode)
391 return NULL;
392
393 /* Copy the Unicode data into the new object */
394 if (u != NULL)
395 Py_UNICODE_COPY(unicode->str, u, size);
396
397 return (PyObject *)unicode;
398}
399
400#ifdef HAVE_WCHAR_H
401
402PyObject *PyUnicode_FromWideChar(register const wchar_t *w,
403 Py_ssize_t size)
404{
405 PyUnicodeObject *unicode;
406
407 if (w == NULL) {
408 PyErr_BadInternalCall();
409 return NULL;
410 }
411
412 unicode = _PyUnicode_New(size);
413 if (!unicode)
414 return NULL;
415
416 /* Copy the wchar_t data into the new object */
417#ifdef HAVE_USABLE_WCHAR_T
418 memcpy(unicode->str, w, size * sizeof(wchar_t));
419#else
420 {
421 register Py_UNICODE *u;
422 register Py_ssize_t i;
423 u = PyUnicode_AS_UNICODE(unicode);
424 for (i = size; i > 0; i--)
425 *u++ = *w++;
426 }
427#endif
428
429 return (PyObject *)unicode;
430}
431
432Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *unicode,
433 wchar_t *w,
434 Py_ssize_t size)
435{
436 if (unicode == NULL) {
437 PyErr_BadInternalCall();
438 return -1;
439 }
440
441 /* If possible, try to copy the 0-termination as well */
442 if (size > PyUnicode_GET_SIZE(unicode))
443 size = PyUnicode_GET_SIZE(unicode) + 1;
444
445#ifdef HAVE_USABLE_WCHAR_T
446 memcpy(w, unicode->str, size * sizeof(wchar_t));
447#else
448 {
449 register Py_UNICODE *u;
450 register Py_ssize_t i;
451 u = PyUnicode_AS_UNICODE(unicode);
452 for (i = size; i > 0; i--)
453 *w++ = *u++;
454 }
455#endif
456
457 if (size > PyUnicode_GET_SIZE(unicode))
458 return PyUnicode_GET_SIZE(unicode);
459 else
460 return size;
461}
462
463#endif
464
465PyObject *PyUnicode_FromOrdinal(int ordinal)
466{
467 Py_UNICODE s[1];
468
469#ifdef Py_UNICODE_WIDE
470 if (ordinal < 0 || ordinal > 0x10ffff) {
471 PyErr_SetString(PyExc_ValueError,
472 "unichr() arg not in range(0x110000) "
473 "(wide Python build)");
474 return NULL;
475 }
476#else
477 if (ordinal < 0 || ordinal > 0xffff) {
478 PyErr_SetString(PyExc_ValueError,
479 "unichr() arg not in range(0x10000) "
480 "(narrow Python build)");
481 return NULL;
482 }
483#endif
484
485 s[0] = (Py_UNICODE)ordinal;
486 return PyUnicode_FromUnicode(s, 1);
487}
488
489PyObject *PyUnicode_FromObject(register PyObject *obj)
490{
491 /* XXX Perhaps we should make this API an alias of
492 PyObject_Unicode() instead ?! */
493 if (PyUnicode_CheckExact(obj)) {
494 Py_INCREF(obj);
495 return obj;
496 }
497 if (PyUnicode_Check(obj)) {
498 /* For a Unicode subtype that's not a Unicode object,
499 return a true Unicode object with the same data. */
500 return PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(obj),
501 PyUnicode_GET_SIZE(obj));
502 }
503 return PyUnicode_FromEncodedObject(obj, NULL, "strict");
504}
505
506PyObject *PyUnicode_FromEncodedObject(register PyObject *obj,
507 const char *encoding,
508 const char *errors)
509{
510 const char *s = NULL;
511 Py_ssize_t len;
512 PyObject *v;
513
514 if (obj == NULL) {
515 PyErr_BadInternalCall();
516 return NULL;
517 }
518
519#if 0
520 /* For b/w compatibility we also accept Unicode objects provided
521 that no encodings is given and then redirect to
522 PyObject_Unicode() which then applies the additional logic for
523 Unicode subclasses.
524
525 NOTE: This API should really only be used for object which
526 represent *encoded* Unicode !
527
528 */
529 if (PyUnicode_Check(obj)) {
530 if (encoding) {
531 PyErr_SetString(PyExc_TypeError,
532 "decoding Unicode is not supported");
533 return NULL;
534 }
535 return PyObject_Unicode(obj);
536 }
537#else
538 if (PyUnicode_Check(obj)) {
539 PyErr_SetString(PyExc_TypeError,
540 "decoding Unicode is not supported");
541 return NULL;
542 }
543#endif
544
545 /* Coerce object */
546 if (PyString_Check(obj)) {
547 s = PyString_AS_STRING(obj);
548 len = PyString_GET_SIZE(obj);
549 }
550 else if (PyObject_AsCharBuffer(obj, &s, &len)) {
551 /* Overwrite the error message with something more useful in
552 case of a TypeError. */
553 if (PyErr_ExceptionMatches(PyExc_TypeError))
554 PyErr_Format(PyExc_TypeError,
555 "coercing to Unicode: need string or buffer, "
556 "%.80s found",
557 obj->ob_type->tp_name);
558 goto onError;
559 }
560
561 /* Convert to Unicode */
562 if (len == 0) {
563 Py_INCREF(unicode_empty);
564 v = (PyObject *)unicode_empty;
565 }
566 else
567 v = PyUnicode_Decode(s, len, encoding, errors);
568
569 return v;
570
571 onError:
572 return NULL;
573}
574
575PyObject *PyUnicode_Decode(const char *s,
576 Py_ssize_t size,
577 const char *encoding,
578 const char *errors)
579{
580 PyObject *buffer = NULL, *unicode;
581
582 if (encoding == NULL)
583 encoding = PyUnicode_GetDefaultEncoding();
584
585 /* Shortcuts for common default encodings */
586 if (strcmp(encoding, "utf-8") == 0)
587 return PyUnicode_DecodeUTF8(s, size, errors);
588 else if (strcmp(encoding, "latin-1") == 0)
589 return PyUnicode_DecodeLatin1(s, size, errors);
590#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
591 else if (strcmp(encoding, "mbcs") == 0)
592 return PyUnicode_DecodeMBCS(s, size, errors);
593#endif
594 else if (strcmp(encoding, "ascii") == 0)
595 return PyUnicode_DecodeASCII(s, size, errors);
596
597 /* Decode via the codec registry */
598 buffer = PyBuffer_FromMemory((void *)s, size);
599 if (buffer == NULL)
600 goto onError;
601 unicode = PyCodec_Decode(buffer, encoding, errors);
602 if (unicode == NULL)
603 goto onError;
604 if (!PyUnicode_Check(unicode)) {
605 PyErr_Format(PyExc_TypeError,
606 "decoder did not return an unicode object (type=%.400s)",
607 unicode->ob_type->tp_name);
608 Py_DECREF(unicode);
609 goto onError;
610 }
611 Py_DECREF(buffer);
612 return unicode;
613
614 onError:
615 Py_XDECREF(buffer);
616 return NULL;
617}
618
619PyObject *PyUnicode_AsDecodedObject(PyObject *unicode,
620 const char *encoding,
621 const char *errors)
622{
623 PyObject *v;
624
625 if (!PyUnicode_Check(unicode)) {
626 PyErr_BadArgument();
627 goto onError;
628 }
629
630 if (encoding == NULL)
631 encoding = PyUnicode_GetDefaultEncoding();
632
633 /* Decode via the codec registry */
634 v = PyCodec_Decode(unicode, encoding, errors);
635 if (v == NULL)
636 goto onError;
637 return v;
638
639 onError:
640 return NULL;
641}
642
643PyObject *PyUnicode_Encode(const Py_UNICODE *s,
644 Py_ssize_t size,
645 const char *encoding,
646 const char *errors)
647{
648 PyObject *v, *unicode;
649
650 unicode = PyUnicode_FromUnicode(s, size);
651 if (unicode == NULL)
652 return NULL;
653 v = PyUnicode_AsEncodedString(unicode, encoding, errors);
654 Py_DECREF(unicode);
655 return v;
656}
657
658PyObject *PyUnicode_AsEncodedObject(PyObject *unicode,
659 const char *encoding,
660 const char *errors)
661{
662 PyObject *v;
663
664 if (!PyUnicode_Check(unicode)) {
665 PyErr_BadArgument();
666 goto onError;
667 }
668
669 if (encoding == NULL)
670 encoding = PyUnicode_GetDefaultEncoding();
671
672 /* Encode via the codec registry */
673 v = PyCodec_Encode(unicode, encoding, errors);
674 if (v == NULL)
675 goto onError;
676 return v;
677
678 onError:
679 return NULL;
680}
681
682PyObject *PyUnicode_AsEncodedString(PyObject *unicode,
683 const char *encoding,
684 const char *errors)
685{
686 PyObject *v;
687
688 if (!PyUnicode_Check(unicode)) {
689 PyErr_BadArgument();
690 goto onError;
691 }
692
693 if (encoding == NULL)
694 encoding = PyUnicode_GetDefaultEncoding();
695
696 /* Shortcuts for common default encodings */
697 if (errors == NULL) {
698 if (strcmp(encoding, "utf-8") == 0)
699 return PyUnicode_AsUTF8String(unicode);
700 else if (strcmp(encoding, "latin-1") == 0)
701 return PyUnicode_AsLatin1String(unicode);
702#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
703 else if (strcmp(encoding, "mbcs") == 0)
704 return PyUnicode_AsMBCSString(unicode);
705#endif
706 else if (strcmp(encoding, "ascii") == 0)
707 return PyUnicode_AsASCIIString(unicode);
708 }
709
710 /* Encode via the codec registry */
711 v = PyCodec_Encode(unicode, encoding, errors);
712 if (v == NULL)
713 goto onError;
714 if (!PyString_Check(v)) {
715 PyErr_Format(PyExc_TypeError,
716 "encoder did not return a string object (type=%.400s)",
717 v->ob_type->tp_name);
718 Py_DECREF(v);
719 goto onError;
720 }
721 return v;
722
723 onError:
724 return NULL;
725}
726
727PyObject *_PyUnicode_AsDefaultEncodedString(PyObject *unicode,
728 const char *errors)
729{
730 PyObject *v = ((PyUnicodeObject *)unicode)->defenc;
731
732 if (v)
733 return v;
734 v = PyUnicode_AsEncodedString(unicode, NULL, errors);
735 if (v && errors == NULL)
736 ((PyUnicodeObject *)unicode)->defenc = v;
737 return v;
738}
739
740Py_UNICODE *PyUnicode_AsUnicode(PyObject *unicode)
741{
742 if (!PyUnicode_Check(unicode)) {
743 PyErr_BadArgument();
744 goto onError;
745 }
746 return PyUnicode_AS_UNICODE(unicode);
747
748 onError:
749 return NULL;
750}
751
752Py_ssize_t PyUnicode_GetSize(PyObject *unicode)
753{
754 if (!PyUnicode_Check(unicode)) {
755 PyErr_BadArgument();
756 goto onError;
757 }
758 return PyUnicode_GET_SIZE(unicode);
759
760 onError:
761 return -1;
762}
763
764const char *PyUnicode_GetDefaultEncoding(void)
765{
766 return unicode_default_encoding;
767}
768
769int PyUnicode_SetDefaultEncoding(const char *encoding)
770{
771 PyObject *v;
772
773 /* Make sure the encoding is valid. As side effect, this also
774 loads the encoding into the codec registry cache. */
775 v = _PyCodec_Lookup(encoding);
776 if (v == NULL)
777 goto onError;
778 Py_DECREF(v);
779 strncpy(unicode_default_encoding,
780 encoding,
781 sizeof(unicode_default_encoding));
782 return 0;
783
784 onError:
785 return -1;
786}
787
788/* error handling callback helper:
789 build arguments, call the callback and check the arguments,
790 if no exception occurred, copy the replacement to the output
791 and adjust various state variables.
792 return 0 on success, -1 on error
793*/
794
795static
796int unicode_decode_call_errorhandler(const char *errors, PyObject **errorHandler,
797 const char *encoding, const char *reason,
798 const char *input, Py_ssize_t insize, Py_ssize_t *startinpos, Py_ssize_t *endinpos, PyObject **exceptionObject, const char **inptr,
799 PyObject **output, Py_ssize_t *outpos, Py_UNICODE **outptr)
800{
801 static char *argparse = "O!n;decoding error handler must return (unicode, int) tuple";
802
803 PyObject *restuple = NULL;
804 PyObject *repunicode = NULL;
805 Py_ssize_t outsize = PyUnicode_GET_SIZE(*output);
806 Py_ssize_t requiredsize;
807 Py_ssize_t newpos;
808 Py_UNICODE *repptr;
809 Py_ssize_t repsize;
810 int res = -1;
811
812 if (*errorHandler == NULL) {
813 *errorHandler = PyCodec_LookupError(errors);
814 if (*errorHandler == NULL)
815 goto onError;
816 }
817
818 if (*exceptionObject == NULL) {
819 *exceptionObject = PyUnicodeDecodeError_Create(
820 encoding, input, insize, *startinpos, *endinpos, reason);
821 if (*exceptionObject == NULL)
822 goto onError;
823 }
824 else {
825 if (PyUnicodeDecodeError_SetStart(*exceptionObject, *startinpos))
826 goto onError;
827 if (PyUnicodeDecodeError_SetEnd(*exceptionObject, *endinpos))
828 goto onError;
829 if (PyUnicodeDecodeError_SetReason(*exceptionObject, reason))
830 goto onError;
831 }
832
833 restuple = PyObject_CallFunctionObjArgs(*errorHandler, *exceptionObject, NULL);
834 if (restuple == NULL)
835 goto onError;
836 if (!PyTuple_Check(restuple)) {
837 PyErr_Format(PyExc_TypeError, &argparse[4]);
838 goto onError;
839 }
840 if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type, &repunicode, &newpos))
841 goto onError;
842 if (newpos<0)
843 newpos = insize+newpos;
844 if (newpos<0 || newpos>insize) {
845 PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", newpos);
846 goto onError;
847 }
848
849 /* need more space? (at least enough for what we
850 have+the replacement+the rest of the string (starting
851 at the new input position), so we won't have to check space
852 when there are no errors in the rest of the string) */
853 repptr = PyUnicode_AS_UNICODE(repunicode);
854 repsize = PyUnicode_GET_SIZE(repunicode);
855 requiredsize = *outpos + repsize + insize-newpos;
856 if (requiredsize > outsize) {
857 if (requiredsize<2*outsize)
858 requiredsize = 2*outsize;
859 if (PyUnicode_Resize(output, requiredsize) < 0)
860 goto onError;
861 *outptr = PyUnicode_AS_UNICODE(*output) + *outpos;
862 }
863 *endinpos = newpos;
864 *inptr = input + newpos;
865 Py_UNICODE_COPY(*outptr, repptr, repsize);
866 *outptr += repsize;
867 *outpos += repsize;
868 /* we made it! */
869 res = 0;
870
871 onError:
872 Py_XDECREF(restuple);
873 return res;
874}
875
876/* --- UTF-7 Codec -------------------------------------------------------- */
877
878/* see RFC2152 for details */
879
880static
881char utf7_special[128] = {
882 /* indicate whether a UTF-7 character is special i.e. cannot be directly
883 encoded:
884 0 - not special
885 1 - special
886 2 - whitespace (optional)
887 3 - RFC2152 Set O (optional) */
888 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 1, 1, 2, 1, 1,
889 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
890 2, 3, 3, 3, 3, 3, 3, 0, 0, 0, 3, 1, 0, 0, 0, 1,
891 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 3, 0,
892 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
893 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3,
894 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
895 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 3, 1, 1,
896
897};
898
899/* Note: The comparison (c) <= 0 is a trick to work-around gcc
900 warnings about the comparison always being false; since
901 utf7_special[0] is 1, we can safely make that one comparison
902 true */
903
904#define SPECIAL(c, encodeO, encodeWS) \
905 ((c) > 127 || (c) <= 0 || utf7_special[(c)] == 1 || \
906 (encodeWS && (utf7_special[(c)] == 2)) || \
907 (encodeO && (utf7_special[(c)] == 3)))
908
909#define B64(n) \
910 ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[(n) & 0x3f])
911#define B64CHAR(c) \
912 (isalnum(c) || (c) == '+' || (c) == '/')
913#define UB64(c) \
914 ((c) == '+' ? 62 : (c) == '/' ? 63 : (c) >= 'a' ? \
915 (c) - 71 : (c) >= 'A' ? (c) - 65 : (c) + 4 )
916
917#define ENCODE(out, ch, bits) \
918 while (bits >= 6) { \
919 *out++ = B64(ch >> (bits-6)); \
920 bits -= 6; \
921 }
922
923#define DECODE(out, ch, bits, surrogate) \
924 while (bits >= 16) { \
925 Py_UNICODE outCh = (Py_UNICODE) ((ch >> (bits-16)) & 0xffff); \
926 bits -= 16; \
927 if (surrogate) { \
928 /* We have already generated an error for the high surrogate \
929 so let's not bother seeing if the low surrogate is correct or not */ \
930 surrogate = 0; \
931 } else if (0xDC00 <= outCh && outCh <= 0xDFFF) { \
932 /* This is a surrogate pair. Unfortunately we can't represent \
933 it in a 16-bit character */ \
934 surrogate = 1; \
935 errmsg = "code pairs are not supported"; \
936 goto utf7Error; \
937 } else { \
938 *out++ = outCh; \
939 } \
940 }
941
942PyObject *PyUnicode_DecodeUTF7(const char *s,
943 Py_ssize_t size,
944 const char *errors)
945{
946 const char *starts = s;
947 Py_ssize_t startinpos;
948 Py_ssize_t endinpos;
949 Py_ssize_t outpos;
950 const char *e;
951 PyUnicodeObject *unicode;
952 Py_UNICODE *p;
953 const char *errmsg = "";
954 int inShift = 0;
955 unsigned int bitsleft = 0;
956 unsigned long charsleft = 0;
957 int surrogate = 0;
958 PyObject *errorHandler = NULL;
959 PyObject *exc = NULL;
960
961 unicode = _PyUnicode_New(size);
962 if (!unicode)
963 return NULL;
964 if (size == 0)
965 return (PyObject *)unicode;
966
967 p = unicode->str;
968 e = s + size;
969
970 while (s < e) {
971 Py_UNICODE ch;
972 restart:
973 ch = *s;
974
975 if (inShift) {
976 if ((ch == '-') || !B64CHAR(ch)) {
977 inShift = 0;
978 s++;
979
980 /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
981 if (bitsleft >= 6) {
982 /* The shift sequence has a partial character in it. If
983 bitsleft < 6 then we could just classify it as padding
984 but that is not the case here */
985
986 errmsg = "partial character in shift sequence";
987 goto utf7Error;
988 }
989 /* According to RFC2152 the remaining bits should be zero. We
990 choose to signal an error/insert a replacement character
991 here so indicate the potential of a misencoded character. */
992
993 /* On x86, a << b == a << (b%32) so make sure that bitsleft != 0 */
994 if (bitsleft && charsleft << (sizeof(charsleft) * 8 - bitsleft)) {
995 errmsg = "non-zero padding bits in shift sequence";
996 goto utf7Error;
997 }
998
999 if (ch == '-') {
1000 if ((s < e) && (*(s) == '-')) {
1001 *p++ = '-';
1002 inShift = 1;
1003 }
1004 } else if (SPECIAL(ch,0,0)) {
1005 errmsg = "unexpected special character";
1006 goto utf7Error;
1007 } else {
1008 *p++ = ch;
1009 }
1010 } else {
1011 charsleft = (charsleft << 6) | UB64(ch);
1012 bitsleft += 6;
1013 s++;
1014 /* p, charsleft, bitsleft, surrogate = */ DECODE(p, charsleft, bitsleft, surrogate);
1015 }
1016 }
1017 else if ( ch == '+' ) {
1018 startinpos = s-starts;
1019 s++;
1020 if (s < e && *s == '-') {
1021 s++;
1022 *p++ = '+';
1023 } else
1024 {
1025 inShift = 1;
1026 bitsleft = 0;
1027 }
1028 }
1029 else if (SPECIAL(ch,0,0)) {
1030 errmsg = "unexpected special character";
1031 s++;
1032 goto utf7Error;
1033 }
1034 else {
1035 *p++ = ch;
1036 s++;
1037 }
1038 continue;
1039 utf7Error:
1040 outpos = p-PyUnicode_AS_UNICODE(unicode);
1041 endinpos = s-starts;
1042 if (unicode_decode_call_errorhandler(
1043 errors, &errorHandler,
1044 "utf7", errmsg,
1045 starts, size, &startinpos, &endinpos, &exc, &s,
1046 (PyObject **)&unicode, &outpos, &p))
1047 goto onError;
1048 }
1049
1050 if (inShift) {
1051 outpos = p-PyUnicode_AS_UNICODE(unicode);
1052 endinpos = size;
1053 if (unicode_decode_call_errorhandler(
1054 errors, &errorHandler,
1055 "utf7", "unterminated shift sequence",
1056 starts, size, &startinpos, &endinpos, &exc, &s,
1057 (PyObject **)&unicode, &outpos, &p))
1058 goto onError;
1059 if (s < e)
1060 goto restart;
1061 }
1062
1063 if (_PyUnicode_Resize(&unicode, p - PyUnicode_AS_UNICODE(unicode)) < 0)
1064 goto onError;
1065
1066 Py_XDECREF(errorHandler);
1067 Py_XDECREF(exc);
1068 return (PyObject *)unicode;
1069
1070onError:
1071 Py_XDECREF(errorHandler);
1072 Py_XDECREF(exc);
1073 Py_DECREF(unicode);
1074 return NULL;
1075}
1076
1077
1078PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
1079 Py_ssize_t size,
1080 int encodeSetO,
1081 int encodeWhiteSpace,
1082 const char *errors)
1083{
1084 PyObject *v;
1085 /* It might be possible to tighten this worst case */
1086 Py_ssize_t cbAllocated = 5 * size;
1087 int inShift = 0;
1088 Py_ssize_t i = 0;
1089 unsigned int bitsleft = 0;
1090 unsigned long charsleft = 0;
1091 char * out;
1092 char * start;
1093
1094 if (size == 0)
1095 return PyString_FromStringAndSize(NULL, 0);
1096
1097 v = PyString_FromStringAndSize(NULL, cbAllocated);
1098 if (v == NULL)
1099 return NULL;
1100
1101 start = out = PyString_AS_STRING(v);
1102 for (;i < size; ++i) {
1103 Py_UNICODE ch = s[i];
1104
1105 if (!inShift) {
1106 if (ch == '+') {
1107 *out++ = '+';
1108 *out++ = '-';
1109 } else if (SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
1110 charsleft = ch;
1111 bitsleft = 16;
1112 *out++ = '+';
1113 /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
1114 inShift = bitsleft > 0;
1115 } else {
1116 *out++ = (char) ch;
1117 }
1118 } else {
1119 if (!SPECIAL(ch, encodeSetO, encodeWhiteSpace)) {
1120 *out++ = B64(charsleft << (6-bitsleft));
1121 charsleft = 0;
1122 bitsleft = 0;
1123 /* Characters not in the BASE64 set implicitly unshift the sequence
1124 so no '-' is required, except if the character is itself a '-' */
1125 if (B64CHAR(ch) || ch == '-') {
1126 *out++ = '-';
1127 }
1128 inShift = 0;
1129 *out++ = (char) ch;
1130 } else {
1131 bitsleft += 16;
1132 charsleft = (charsleft << 16) | ch;
1133 /* out, charsleft, bitsleft = */ ENCODE(out, charsleft, bitsleft);
1134
1135 /* If the next character is special then we dont' need to terminate
1136 the shift sequence. If the next character is not a BASE64 character
1137 or '-' then the shift sequence will be terminated implicitly and we
1138 don't have to insert a '-'. */
1139
1140 if (bitsleft == 0) {
1141 if (i + 1 < size) {
1142 Py_UNICODE ch2 = s[i+1];
1143
1144 if (SPECIAL(ch2, encodeSetO, encodeWhiteSpace)) {
1145
1146 } else if (B64CHAR(ch2) || ch2 == '-') {
1147 *out++ = '-';
1148 inShift = 0;
1149 } else {
1150 inShift = 0;
1151 }
1152
1153 }
1154 else {
1155 *out++ = '-';
1156 inShift = 0;
1157 }
1158 }
1159 }
1160 }
1161 }
1162 if (bitsleft) {
1163 *out++= B64(charsleft << (6-bitsleft) );
1164 *out++ = '-';
1165 }
1166
1167 _PyString_Resize(&v, out - start);
1168 return v;
1169}
1170
1171#undef SPECIAL
1172#undef B64
1173#undef B64CHAR
1174#undef UB64
1175#undef ENCODE
1176#undef DECODE
1177
1178/* --- UTF-8 Codec -------------------------------------------------------- */
1179
1180static
1181char utf8_code_length[256] = {
1182 /* Map UTF-8 encoded prefix byte to sequence length. zero means
1183 illegal prefix. see RFC 2279 for details */
1184 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1185 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1186 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1187 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1188 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1189 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1190 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1191 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
1192 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1193 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1194 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1195 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1196 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1197 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
1198 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
1199 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0
1200};
1201
1202PyObject *PyUnicode_DecodeUTF8(const char *s,
1203 Py_ssize_t size,
1204 const char *errors)
1205{
1206 return PyUnicode_DecodeUTF8Stateful(s, size, errors, NULL);
1207}
1208
1209PyObject *PyUnicode_DecodeUTF8Stateful(const char *s,
1210 Py_ssize_t size,
1211 const char *errors,
1212 Py_ssize_t *consumed)
1213{
1214 const char *starts = s;
1215 int n;
1216 Py_ssize_t startinpos;
1217 Py_ssize_t endinpos;
1218 Py_ssize_t outpos;
1219 const char *e;
1220 PyUnicodeObject *unicode;
1221 Py_UNICODE *p;
1222 const char *errmsg = "";
1223 PyObject *errorHandler = NULL;
1224 PyObject *exc = NULL;
1225
1226 /* Note: size will always be longer than the resulting Unicode
1227 character count */
1228 unicode = _PyUnicode_New(size);
1229 if (!unicode)
1230 return NULL;
1231 if (size == 0) {
1232 if (consumed)
1233 *consumed = 0;
1234 return (PyObject *)unicode;
1235 }
1236
1237 /* Unpack UTF-8 encoded data */
1238 p = unicode->str;
1239 e = s + size;
1240
1241 while (s < e) {
1242 Py_UCS4 ch = (unsigned char)*s;
1243
1244 if (ch < 0x80) {
1245 *p++ = (Py_UNICODE)ch;
1246 s++;
1247 continue;
1248 }
1249
1250 n = utf8_code_length[ch];
1251
1252 if (s + n > e) {
1253 if (consumed)
1254 break;
1255 else {
1256 errmsg = "unexpected end of data";
1257 startinpos = s-starts;
1258 endinpos = size;
1259 goto utf8Error;
1260 }
1261 }
1262
1263 switch (n) {
1264
1265 case 0:
1266 errmsg = "unexpected code byte";
1267 startinpos = s-starts;
1268 endinpos = startinpos+1;
1269 goto utf8Error;
1270
1271 case 1:
1272 errmsg = "internal error";
1273 startinpos = s-starts;
1274 endinpos = startinpos+1;
1275 goto utf8Error;
1276
1277 case 2:
1278 if ((s[1] & 0xc0) != 0x80) {
1279 errmsg = "invalid data";
1280 startinpos = s-starts;
1281 endinpos = startinpos+2;
1282 goto utf8Error;
1283 }
1284 ch = ((s[0] & 0x1f) << 6) + (s[1] & 0x3f);
1285 if (ch < 0x80) {
1286 startinpos = s-starts;
1287 endinpos = startinpos+2;
1288 errmsg = "illegal encoding";
1289 goto utf8Error;
1290 }
1291 else
1292 *p++ = (Py_UNICODE)ch;
1293 break;
1294
1295 case 3:
1296 if ((s[1] & 0xc0) != 0x80 ||
1297 (s[2] & 0xc0) != 0x80) {
1298 errmsg = "invalid data";
1299 startinpos = s-starts;
1300 endinpos = startinpos+3;
1301 goto utf8Error;
1302 }
1303 ch = ((s[0] & 0x0f) << 12) + ((s[1] & 0x3f) << 6) + (s[2] & 0x3f);
1304 if (ch < 0x0800) {
1305 /* Note: UTF-8 encodings of surrogates are considered
1306 legal UTF-8 sequences;
1307
1308 XXX For wide builds (UCS-4) we should probably try
1309 to recombine the surrogates into a single code
1310 unit.
1311 */
1312 errmsg = "illegal encoding";
1313 startinpos = s-starts;
1314 endinpos = startinpos+3;
1315 goto utf8Error;
1316 }
1317 else
1318 *p++ = (Py_UNICODE)ch;
1319 break;
1320
1321 case 4:
1322 if ((s[1] & 0xc0) != 0x80 ||
1323 (s[2] & 0xc0) != 0x80 ||
1324 (s[3] & 0xc0) != 0x80) {
1325 errmsg = "invalid data";
1326 startinpos = s-starts;
1327 endinpos = startinpos+4;
1328 goto utf8Error;
1329 }
1330 ch = ((s[0] & 0x7) << 18) + ((s[1] & 0x3f) << 12) +
1331 ((s[2] & 0x3f) << 6) + (s[3] & 0x3f);
1332 /* validate and convert to UTF-16 */
1333 if ((ch < 0x10000) /* minimum value allowed for 4
1334 byte encoding */
1335 || (ch > 0x10ffff)) /* maximum value allowed for
1336 UTF-16 */
1337 {
1338 errmsg = "illegal encoding";
1339 startinpos = s-starts;
1340 endinpos = startinpos+4;
1341 goto utf8Error;
1342 }
1343#ifdef Py_UNICODE_WIDE
1344 *p++ = (Py_UNICODE)ch;
1345#else
1346 /* compute and append the two surrogates: */
1347
1348 /* translate from 10000..10FFFF to 0..FFFF */
1349 ch -= 0x10000;
1350
1351 /* high surrogate = top 10 bits added to D800 */
1352 *p++ = (Py_UNICODE)(0xD800 + (ch >> 10));
1353
1354 /* low surrogate = bottom 10 bits added to DC00 */
1355 *p++ = (Py_UNICODE)(0xDC00 + (ch & 0x03FF));
1356#endif
1357 break;
1358
1359 default:
1360 /* Other sizes are only needed for UCS-4 */
1361 errmsg = "unsupported Unicode code range";
1362 startinpos = s-starts;
1363 endinpos = startinpos+n;
1364 goto utf8Error;
1365 }
1366 s += n;
1367 continue;
1368
1369 utf8Error:
1370 outpos = p-PyUnicode_AS_UNICODE(unicode);
1371 if (unicode_decode_call_errorhandler(
1372 errors, &errorHandler,
1373 "utf8", errmsg,
1374 starts, size, &startinpos, &endinpos, &exc, &s,
1375 (PyObject **)&unicode, &outpos, &p))
1376 goto onError;
1377 }
1378 if (consumed)
1379 *consumed = s-starts;
1380
1381 /* Adjust length */
1382 if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0)
1383 goto onError;
1384
1385 Py_XDECREF(errorHandler);
1386 Py_XDECREF(exc);
1387 return (PyObject *)unicode;
1388
1389onError:
1390 Py_XDECREF(errorHandler);
1391 Py_XDECREF(exc);
1392 Py_DECREF(unicode);
1393 return NULL;
1394}
1395
1396/* Allocation strategy: if the string is short, convert into a stack buffer
1397 and allocate exactly as much space needed at the end. Else allocate the
1398 maximum possible needed (4 result bytes per Unicode character), and return
1399 the excess memory at the end.
1400*/
1401PyObject *
1402PyUnicode_EncodeUTF8(const Py_UNICODE *s,
1403 Py_ssize_t size,
1404 const char *errors)
1405{
1406#define MAX_SHORT_UNICHARS 300 /* largest size we'll do on the stack */
1407
1408 Py_ssize_t i; /* index into s of next input byte */
1409 PyObject *v; /* result string object */
1410 char *p; /* next free byte in output buffer */
1411 Py_ssize_t nallocated; /* number of result bytes allocated */
1412 Py_ssize_t nneeded; /* number of result bytes needed */
1413 char stackbuf[MAX_SHORT_UNICHARS * 4];
1414
1415 assert(s != NULL);
1416 assert(size >= 0);
1417
1418 if (size <= MAX_SHORT_UNICHARS) {
1419 /* Write into the stack buffer; nallocated can't overflow.
1420 * At the end, we'll allocate exactly as much heap space as it
1421 * turns out we need.
1422 */
1423 nallocated = Py_SAFE_DOWNCAST(sizeof(stackbuf), size_t, int);
1424 v = NULL; /* will allocate after we're done */
1425 p = stackbuf;
1426 }
1427 else {
1428 /* Overallocate on the heap, and give the excess back at the end. */
1429 nallocated = size * 4;
1430 if (nallocated / 4 != size) /* overflow! */
1431 return PyErr_NoMemory();
1432 v = PyString_FromStringAndSize(NULL, nallocated);
1433 if (v == NULL)
1434 return NULL;
1435 p = PyString_AS_STRING(v);
1436 }
1437
1438 for (i = 0; i < size;) {
1439 Py_UCS4 ch = s[i++];
1440
1441 if (ch < 0x80)
1442 /* Encode ASCII */
1443 *p++ = (char) ch;
1444
1445 else if (ch < 0x0800) {
1446 /* Encode Latin-1 */
1447 *p++ = (char)(0xc0 | (ch >> 6));
1448 *p++ = (char)(0x80 | (ch & 0x3f));
1449 }
1450 else {
1451 /* Encode UCS2 Unicode ordinals */
1452 if (ch < 0x10000) {
1453 /* Special case: check for high surrogate */
1454 if (0xD800 <= ch && ch <= 0xDBFF && i != size) {
1455 Py_UCS4 ch2 = s[i];
1456 /* Check for low surrogate and combine the two to
1457 form a UCS4 value */
1458 if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
1459 ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;
1460 i++;
1461 goto encodeUCS4;
1462 }
1463 /* Fall through: handles isolated high surrogates */
1464 }
1465 *p++ = (char)(0xe0 | (ch >> 12));
1466 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
1467 *p++ = (char)(0x80 | (ch & 0x3f));
1468 continue;
1469 }
1470encodeUCS4:
1471 /* Encode UCS4 Unicode ordinals */
1472 *p++ = (char)(0xf0 | (ch >> 18));
1473 *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
1474 *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
1475 *p++ = (char)(0x80 | (ch & 0x3f));
1476 }
1477 }
1478
1479 if (v == NULL) {
1480 /* This was stack allocated. */
1481 nneeded = p - stackbuf;
1482 assert(nneeded <= nallocated);
1483 v = PyString_FromStringAndSize(stackbuf, nneeded);
1484 }
1485 else {
1486 /* Cut back to size actually needed. */
1487 nneeded = p - PyString_AS_STRING(v);
1488 assert(nneeded <= nallocated);
1489 _PyString_Resize(&v, nneeded);
1490 }
1491 return v;
1492
1493#undef MAX_SHORT_UNICHARS
1494}
1495
1496PyObject *PyUnicode_AsUTF8String(PyObject *unicode)
1497{
1498 if (!PyUnicode_Check(unicode)) {
1499 PyErr_BadArgument();
1500 return NULL;
1501 }
1502 return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(unicode),
1503 PyUnicode_GET_SIZE(unicode),
1504 NULL);
1505}
1506
1507/* --- UTF-16 Codec ------------------------------------------------------- */
1508
1509PyObject *
1510PyUnicode_DecodeUTF16(const char *s,
1511 Py_ssize_t size,
1512 const char *errors,
1513 int *byteorder)
1514{
1515 return PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder, NULL);
1516}
1517
1518PyObject *
1519PyUnicode_DecodeUTF16Stateful(const char *s,
1520 Py_ssize_t size,
1521 const char *errors,
1522 int *byteorder,
1523 Py_ssize_t *consumed)
1524{
1525 const char *starts = s;
1526 Py_ssize_t startinpos;
1527 Py_ssize_t endinpos;
1528 Py_ssize_t outpos;
1529 PyUnicodeObject *unicode;
1530 Py_UNICODE *p;
1531 const unsigned char *q, *e;
1532 int bo = 0; /* assume native ordering by default */
1533 const char *errmsg = "";
1534 /* Offsets from q for retrieving byte pairs in the right order. */
1535#ifdef BYTEORDER_IS_LITTLE_ENDIAN
1536 int ihi = 1, ilo = 0;
1537#else
1538 int ihi = 0, ilo = 1;
1539#endif
1540 PyObject *errorHandler = NULL;
1541 PyObject *exc = NULL;
1542
1543 /* Note: size will always be longer than the resulting Unicode
1544 character count */
1545 unicode = _PyUnicode_New(size);
1546 if (!unicode)
1547 return NULL;
1548 if (size == 0)
1549 return (PyObject *)unicode;
1550
1551 /* Unpack UTF-16 encoded data */
1552 p = unicode->str;
1553 q = (unsigned char *)s;
1554 e = q + size;
1555
1556 if (byteorder)
1557 bo = *byteorder;
1558
1559 /* Check for BOM marks (U+FEFF) in the input and adjust current
1560 byte order setting accordingly. In native mode, the leading BOM
1561 mark is skipped, in all other modes, it is copied to the output
1562 stream as-is (giving a ZWNBSP character). */
1563 if (bo == 0) {
1564 if (size >= 2) {
1565 const Py_UNICODE bom = (q[ihi] << 8) | q[ilo];
1566#ifdef BYTEORDER_IS_LITTLE_ENDIAN
1567 if (bom == 0xFEFF) {
1568 q += 2;
1569 bo = -1;
1570 }
1571 else if (bom == 0xFFFE) {
1572 q += 2;
1573 bo = 1;
1574 }
1575#else
1576 if (bom == 0xFEFF) {
1577 q += 2;
1578 bo = 1;
1579 }
1580 else if (bom == 0xFFFE) {
1581 q += 2;
1582 bo = -1;
1583 }
1584#endif
1585 }
1586 }
1587
1588 if (bo == -1) {
1589 /* force LE */
1590 ihi = 1;
1591 ilo = 0;
1592 }
1593 else if (bo == 1) {
1594 /* force BE */
1595 ihi = 0;
1596 ilo = 1;
1597 }
1598
1599 while (q < e) {
1600 Py_UNICODE ch;
1601 /* remaining bytes at the end? (size should be even) */
1602 if (e-q<2) {
1603 if (consumed)
1604 break;
1605 errmsg = "truncated data";
1606 startinpos = ((const char *)q)-starts;
1607 endinpos = ((const char *)e)-starts;
1608 goto utf16Error;
1609 /* The remaining input chars are ignored if the callback
1610 chooses to skip the input */
1611 }
1612 ch = (q[ihi] << 8) | q[ilo];
1613
1614 q += 2;
1615
1616 if (ch < 0xD800 || ch > 0xDFFF) {
1617 *p++ = ch;
1618 continue;
1619 }
1620
1621 /* UTF-16 code pair: */
1622 if (q >= e) {
1623 errmsg = "unexpected end of data";
1624 startinpos = (((const char *)q)-2)-starts;
1625 endinpos = ((const char *)e)-starts;
1626 goto utf16Error;
1627 }
1628 if (0xD800 <= ch && ch <= 0xDBFF) {
1629 Py_UNICODE ch2 = (q[ihi] << 8) | q[ilo];
1630 q += 2;
1631 if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
1632#ifndef Py_UNICODE_WIDE
1633 *p++ = ch;
1634 *p++ = ch2;
1635#else
1636 *p++ = (((ch & 0x3FF)<<10) | (ch2 & 0x3FF)) + 0x10000;
1637#endif
1638 continue;
1639 }
1640 else {
1641 errmsg = "illegal UTF-16 surrogate";
1642 startinpos = (((const char *)q)-4)-starts;
1643 endinpos = startinpos+2;
1644 goto utf16Error;
1645 }
1646
1647 }
1648 errmsg = "illegal encoding";
1649 startinpos = (((const char *)q)-2)-starts;
1650 endinpos = startinpos+2;
1651 /* Fall through to report the error */
1652
1653 utf16Error:
1654 outpos = p-PyUnicode_AS_UNICODE(unicode);
1655 if (unicode_decode_call_errorhandler(
1656 errors, &errorHandler,
1657 "utf16", errmsg,
1658 starts, size, &startinpos, &endinpos, &exc, (const char **)&q,
1659 (PyObject **)&unicode, &outpos, &p))
1660 goto onError;
1661 }
1662
1663 if (byteorder)
1664 *byteorder = bo;
1665
1666 if (consumed)
1667 *consumed = (const char *)q-starts;
1668
1669 /* Adjust length */
1670 if (_PyUnicode_Resize(&unicode, p - unicode->str) < 0)
1671 goto onError;
1672
1673 Py_XDECREF(errorHandler);
1674 Py_XDECREF(exc);
1675 return (PyObject *)unicode;
1676
1677onError:
1678 Py_DECREF(unicode);
1679 Py_XDECREF(errorHandler);
1680 Py_XDECREF(exc);
1681 return NULL;
1682}
1683
1684PyObject *
1685PyUnicode_EncodeUTF16(const Py_UNICODE *s,
1686 Py_ssize_t size,
1687 const char *errors,
1688 int byteorder)
1689{
1690 PyObject *v;
1691 unsigned char *p;
1692#ifdef Py_UNICODE_WIDE
1693 int i, pairs;
1694#else
1695 const int pairs = 0;
1696#endif
1697 /* Offsets from p for storing byte pairs in the right order. */
1698#ifdef BYTEORDER_IS_LITTLE_ENDIAN
1699 int ihi = 1, ilo = 0;
1700#else
1701 int ihi = 0, ilo = 1;
1702#endif
1703
1704#define STORECHAR(CH) \
1705 do { \
1706 p[ihi] = ((CH) >> 8) & 0xff; \
1707 p[ilo] = (CH) & 0xff; \
1708 p += 2; \
1709 } while(0)
1710
1711#ifdef Py_UNICODE_WIDE
1712 for (i = pairs = 0; i < size; i++)
1713 if (s[i] >= 0x10000)
1714 pairs++;
1715#endif
1716 v = PyString_FromStringAndSize(NULL,
1717 2 * (size + pairs + (byteorder == 0)));
1718 if (v == NULL)
1719 return NULL;
1720
1721 p = (unsigned char *)PyString_AS_STRING(v);
1722 if (byteorder == 0)
1723 STORECHAR(0xFEFF);
1724 if (size == 0)
1725 return v;
1726
1727 if (byteorder == -1) {
1728 /* force LE */
1729 ihi = 1;
1730 ilo = 0;
1731 }
1732 else if (byteorder == 1) {
1733 /* force BE */
1734 ihi = 0;
1735 ilo = 1;
1736 }
1737
1738 while (size-- > 0) {
1739 Py_UNICODE ch = *s++;
1740 Py_UNICODE ch2 = 0;
1741#ifdef Py_UNICODE_WIDE
1742 if (ch >= 0x10000) {
1743 ch2 = 0xDC00 | ((ch-0x10000) & 0x3FF);
1744 ch = 0xD800 | ((ch-0x10000) >> 10);
1745 }
1746#endif
1747 STORECHAR(ch);
1748 if (ch2)
1749 STORECHAR(ch2);
1750 }
1751 return v;
1752#undef STORECHAR
1753}
1754
1755PyObject *PyUnicode_AsUTF16String(PyObject *unicode)
1756{
1757 if (!PyUnicode_Check(unicode)) {
1758 PyErr_BadArgument();
1759 return NULL;
1760 }
1761 return PyUnicode_EncodeUTF16(PyUnicode_AS_UNICODE(unicode),
1762 PyUnicode_GET_SIZE(unicode),
1763 NULL,
1764 0);
1765}
1766
1767/* --- Unicode Escape Codec ----------------------------------------------- */
1768
1769static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
1770
1771PyObject *PyUnicode_DecodeUnicodeEscape(const char *s,
1772 Py_ssize_t size,
1773 const char *errors)
1774{
1775 const char *starts = s;
1776 Py_ssize_t startinpos;
1777 Py_ssize_t endinpos;
1778 Py_ssize_t outpos;
1779 int i;
1780 PyUnicodeObject *v;
1781 Py_UNICODE *p;
1782 const char *end;
1783 char* message;
1784 Py_UCS4 chr = 0xffffffff; /* in case 'getcode' messes up */
1785 PyObject *errorHandler = NULL;
1786 PyObject *exc = NULL;
1787
1788 /* Escaped strings will always be longer than the resulting
1789 Unicode string, so we start with size here and then reduce the
1790 length after conversion to the true value.
1791 (but if the error callback returns a long replacement string
1792 we'll have to allocate more space) */
1793 v = _PyUnicode_New(size);
1794 if (v == NULL)
1795 goto onError;
1796 if (size == 0)
1797 return (PyObject *)v;
1798
1799 p = PyUnicode_AS_UNICODE(v);
1800 end = s + size;
1801
1802 while (s < end) {
1803 unsigned char c;
1804 Py_UNICODE x;
1805 int digits;
1806
1807 /* Non-escape characters are interpreted as Unicode ordinals */
1808 if (*s != '\\') {
1809 *p++ = (unsigned char) *s++;
1810 continue;
1811 }
1812
1813 startinpos = s-starts;
1814 /* \ - Escapes */
1815 s++;
1816 switch (*s++) {
1817
1818 /* \x escapes */
1819 case '\n': break;
1820 case '\\': *p++ = '\\'; break;
1821 case '\'': *p++ = '\''; break;
1822 case '\"': *p++ = '\"'; break;
1823 case 'b': *p++ = '\b'; break;
1824 case 'f': *p++ = '\014'; break; /* FF */
1825 case 't': *p++ = '\t'; break;
1826 case 'n': *p++ = '\n'; break;
1827 case 'r': *p++ = '\r'; break;
1828 case 'v': *p++ = '\013'; break; /* VT */
1829 case 'a': *p++ = '\007'; break; /* BEL, not classic C */
1830
1831 /* \OOO (octal) escapes */
1832 case '0': case '1': case '2': case '3':
1833 case '4': case '5': case '6': case '7':
1834 x = s[-1] - '0';
1835 if ('0' <= *s && *s <= '7') {
1836 x = (x<<3) + *s++ - '0';
1837 if ('0' <= *s && *s <= '7')
1838 x = (x<<3) + *s++ - '0';
1839 }
1840 *p++ = x;
1841 break;
1842
1843 /* hex escapes */
1844 /* \xXX */
1845 case 'x':
1846 digits = 2;
1847 message = "truncated \\xXX escape";
1848 goto hexescape;
1849
1850 /* \uXXXX */
1851 case 'u':
1852 digits = 4;
1853 message = "truncated \\uXXXX escape";
1854 goto hexescape;
1855
1856 /* \UXXXXXXXX */
1857 case 'U':
1858 digits = 8;
1859 message = "truncated \\UXXXXXXXX escape";
1860 hexescape:
1861 chr = 0;
1862 outpos = p-PyUnicode_AS_UNICODE(v);
1863 if (s+digits>end) {
1864 endinpos = size;
1865 if (unicode_decode_call_errorhandler(
1866 errors, &errorHandler,
1867 "unicodeescape", "end of string in escape sequence",
1868 starts, size, &startinpos, &endinpos, &exc, &s,
1869 (PyObject **)&v, &outpos, &p))
1870 goto onError;
1871 goto nextByte;
1872 }
1873 for (i = 0; i < digits; ++i) {
1874 c = (unsigned char) s[i];
1875 if (!isxdigit(c)) {
1876 endinpos = (s+i+1)-starts;
1877 if (unicode_decode_call_errorhandler(
1878 errors, &errorHandler,
1879 "unicodeescape", message,
1880 starts, size, &startinpos, &endinpos, &exc, &s,
1881 (PyObject **)&v, &outpos, &p))
1882 goto onError;
1883 goto nextByte;
1884 }
1885 chr = (chr<<4) & ~0xF;
1886 if (c >= '0' && c <= '9')
1887 chr += c - '0';
1888 else if (c >= 'a' && c <= 'f')
1889 chr += 10 + c - 'a';
1890 else
1891 chr += 10 + c - 'A';
1892 }
1893 s += i;
1894 if (chr == 0xffffffff && PyErr_Occurred())
1895 /* _decoding_error will have already written into the
1896 target buffer. */
1897 break;
1898 store:
1899 /* when we get here, chr is a 32-bit unicode character */
1900 if (chr <= 0xffff)
1901 /* UCS-2 character */
1902 *p++ = (Py_UNICODE) chr;
1903 else if (chr <= 0x10ffff) {
1904 /* UCS-4 character. Either store directly, or as
1905 surrogate pair. */
1906#ifdef Py_UNICODE_WIDE
1907 *p++ = chr;
1908#else
1909 chr -= 0x10000L;
1910 *p++ = 0xD800 + (Py_UNICODE) (chr >> 10);
1911 *p++ = 0xDC00 + (Py_UNICODE) (chr & 0x03FF);
1912#endif
1913 } else {
1914 endinpos = s-starts;
1915 outpos = p-PyUnicode_AS_UNICODE(v);
1916 if (unicode_decode_call_errorhandler(
1917 errors, &errorHandler,
1918 "unicodeescape", "illegal Unicode character",
1919 starts, size, &startinpos, &endinpos, &exc, &s,
1920 (PyObject **)&v, &outpos, &p))
1921 goto onError;
1922 }
1923 break;
1924
1925 /* \N{name} */
1926 case 'N':
1927 message = "malformed \\N character escape";
1928 if (ucnhash_CAPI == NULL) {
1929 /* load the unicode data module */
1930 PyObject *m, *api;
1931 m = PyImport_ImportModule("unicodedata");
1932 if (m == NULL)
1933 goto ucnhashError;
1934 api = PyObject_GetAttrString(m, "ucnhash_CAPI");
1935 Py_DECREF(m);
1936 if (api == NULL)
1937 goto ucnhashError;
1938 ucnhash_CAPI = (_PyUnicode_Name_CAPI *)PyCObject_AsVoidPtr(api);
1939 Py_DECREF(api);
1940 if (ucnhash_CAPI == NULL)
1941 goto ucnhashError;
1942 }
1943 if (*s == '{') {
1944 const char *start = s+1;
1945 /* look for the closing brace */
1946 while (*s != '}' && s < end)
1947 s++;
1948 if (s > start && s < end && *s == '}') {
1949 /* found a name. look it up in the unicode database */
1950 message = "unknown Unicode character name";
1951 s++;
1952 if (ucnhash_CAPI->getcode(NULL, start, (int)(s-start-1), &chr))
1953 goto store;
1954 }
1955 }
1956 endinpos = s-starts;
1957 outpos = p-PyUnicode_AS_UNICODE(v);
1958 if (unicode_decode_call_errorhandler(
1959 errors, &errorHandler,
1960 "unicodeescape", message,
1961 starts, size, &startinpos, &endinpos, &exc, &s,
1962 (PyObject **)&v, &outpos, &p))
1963 goto onError;
1964 break;
1965
1966 default:
1967 if (s > end) {
1968 message = "\\ at end of string";
1969 s--;
1970 endinpos = s-starts;
1971 outpos = p-PyUnicode_AS_UNICODE(v);
1972 if (unicode_decode_call_errorhandler(
1973 errors, &errorHandler,
1974 "unicodeescape", message,
1975 starts, size, &startinpos, &endinpos, &exc, &s,
1976 (PyObject **)&v, &outpos, &p))
1977 goto onError;
1978 }
1979 else {
1980 *p++ = '\\';
1981 *p++ = (unsigned char)s[-1];
1982 }
1983 break;
1984 }
1985 nextByte:
1986 ;
1987 }
1988 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
1989 goto onError;
1990 Py_XDECREF(errorHandler);
1991 Py_XDECREF(exc);
1992 return (PyObject *)v;
1993
1994ucnhashError:
1995 PyErr_SetString(
1996 PyExc_UnicodeError,
1997 "\\N escapes not supported (can't load unicodedata module)"
1998 );
1999 Py_XDECREF(v);
2000 Py_XDECREF(errorHandler);
2001 Py_XDECREF(exc);
2002 return NULL;
2003
2004onError:
2005 Py_XDECREF(v);
2006 Py_XDECREF(errorHandler);
2007 Py_XDECREF(exc);
2008 return NULL;
2009}
2010
2011/* Return a Unicode-Escape string version of the Unicode object.
2012
2013 If quotes is true, the string is enclosed in u"" or u'' quotes as
2014 appropriate.
2015
2016*/
2017
2018Py_LOCAL_INLINE(const Py_UNICODE *) findchar(const Py_UNICODE *s,
2019 Py_ssize_t size,
2020 Py_UNICODE ch)
2021{
2022 /* like wcschr, but doesn't stop at NULL characters */
2023
2024 while (size-- > 0) {
2025 if (*s == ch)
2026 return s;
2027 s++;
2028 }
2029
2030 return NULL;
2031}
2032
2033static
2034PyObject *unicodeescape_string(const Py_UNICODE *s,
2035 Py_ssize_t size,
2036 int quotes)
2037{
2038 PyObject *repr;
2039 char *p;
2040
2041 static const char *hexdigit = "0123456789abcdef";
2042
2043 /* Initial allocation is based on the longest-possible unichr
2044 escape.
2045
2046 In wide (UTF-32) builds '\U00xxxxxx' is 10 chars per source
2047 unichr, so in this case it's the longest unichr escape. In
2048 narrow (UTF-16) builds this is five chars per source unichr
2049 since there are two unichrs in the surrogate pair, so in narrow
2050 (UTF-16) builds it's not the longest unichr escape.
2051
2052 In wide or narrow builds '\uxxxx' is 6 chars per source unichr,
2053 so in the narrow (UTF-16) build case it's the longest unichr
2054 escape.
2055 */
2056
2057 repr = PyString_FromStringAndSize(NULL,
2058 2
2059#ifdef Py_UNICODE_WIDE
2060 + 10*size
2061#else
2062 + 6*size
2063#endif
2064 + 1);
2065 if (repr == NULL)
2066 return NULL;
2067
2068 p = PyString_AS_STRING(repr);
2069
2070 if (quotes) {
2071 *p++ = 'u';
2072 *p++ = (findchar(s, size, '\'') &&
2073 !findchar(s, size, '"')) ? '"' : '\'';
2074 }
2075 while (size-- > 0) {
2076 Py_UNICODE ch = *s++;
2077
2078 /* Escape quotes and backslashes */
2079 if ((quotes &&
2080 ch == (Py_UNICODE) PyString_AS_STRING(repr)[1]) || ch == '\\') {
2081 *p++ = '\\';
2082 *p++ = (char) ch;
2083 continue;
2084 }
2085
2086#ifdef Py_UNICODE_WIDE
2087 /* Map 21-bit characters to '\U00xxxxxx' */
2088 else if (ch >= 0x10000) {
2089 *p++ = '\\';
2090 *p++ = 'U';
2091 *p++ = hexdigit[(ch >> 28) & 0x0000000F];
2092 *p++ = hexdigit[(ch >> 24) & 0x0000000F];
2093 *p++ = hexdigit[(ch >> 20) & 0x0000000F];
2094 *p++ = hexdigit[(ch >> 16) & 0x0000000F];
2095 *p++ = hexdigit[(ch >> 12) & 0x0000000F];
2096 *p++ = hexdigit[(ch >> 8) & 0x0000000F];
2097 *p++ = hexdigit[(ch >> 4) & 0x0000000F];
2098 *p++ = hexdigit[ch & 0x0000000F];
2099 continue;
2100 }
2101#else
2102 /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */
2103 else if (ch >= 0xD800 && ch < 0xDC00) {
2104 Py_UNICODE ch2;
2105 Py_UCS4 ucs;
2106
2107 ch2 = *s++;
2108 size--;
2109 if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) {
2110 ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000;
2111 *p++ = '\\';
2112 *p++ = 'U';
2113 *p++ = hexdigit[(ucs >> 28) & 0x0000000F];
2114 *p++ = hexdigit[(ucs >> 24) & 0x0000000F];
2115 *p++ = hexdigit[(ucs >> 20) & 0x0000000F];
2116 *p++ = hexdigit[(ucs >> 16) & 0x0000000F];
2117 *p++ = hexdigit[(ucs >> 12) & 0x0000000F];
2118 *p++ = hexdigit[(ucs >> 8) & 0x0000000F];
2119 *p++ = hexdigit[(ucs >> 4) & 0x0000000F];
2120 *p++ = hexdigit[ucs & 0x0000000F];
2121 continue;
2122 }
2123 /* Fall through: isolated surrogates are copied as-is */
2124 s--;
2125 size++;
2126 }
2127#endif
2128
2129 /* Map 16-bit characters to '\uxxxx' */
2130 if (ch >= 256) {
2131 *p++ = '\\';
2132 *p++ = 'u';
2133 *p++ = hexdigit[(ch >> 12) & 0x000F];
2134 *p++ = hexdigit[(ch >> 8) & 0x000F];
2135 *p++ = hexdigit[(ch >> 4) & 0x000F];
2136 *p++ = hexdigit[ch & 0x000F];
2137 }
2138
2139 /* Map special whitespace to '\t', \n', '\r' */
2140 else if (ch == '\t') {
2141 *p++ = '\\';
2142 *p++ = 't';
2143 }
2144 else if (ch == '\n') {
2145 *p++ = '\\';
2146 *p++ = 'n';
2147 }
2148 else if (ch == '\r') {
2149 *p++ = '\\';
2150 *p++ = 'r';
2151 }
2152
2153 /* Map non-printable US ASCII to '\xhh' */
2154 else if (ch < ' ' || ch >= 0x7F) {
2155 *p++ = '\\';
2156 *p++ = 'x';
2157 *p++ = hexdigit[(ch >> 4) & 0x000F];
2158 *p++ = hexdigit[ch & 0x000F];
2159 }
2160
2161 /* Copy everything else as-is */
2162 else
2163 *p++ = (char) ch;
2164 }
2165 if (quotes)
2166 *p++ = PyString_AS_STRING(repr)[1];
2167
2168 *p = '\0';
2169 _PyString_Resize(&repr, p - PyString_AS_STRING(repr));
2170 return repr;
2171}
2172
2173PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
2174 Py_ssize_t size)
2175{
2176 return unicodeescape_string(s, size, 0);
2177}
2178
2179PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
2180{
2181 if (!PyUnicode_Check(unicode)) {
2182 PyErr_BadArgument();
2183 return NULL;
2184 }
2185 return PyUnicode_EncodeUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
2186 PyUnicode_GET_SIZE(unicode));
2187}
2188
2189/* --- Raw Unicode Escape Codec ------------------------------------------- */
2190
2191PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s,
2192 Py_ssize_t size,
2193 const char *errors)
2194{
2195 const char *starts = s;
2196 Py_ssize_t startinpos;
2197 Py_ssize_t endinpos;
2198 Py_ssize_t outpos;
2199 PyUnicodeObject *v;
2200 Py_UNICODE *p;
2201 const char *end;
2202 const char *bs;
2203 PyObject *errorHandler = NULL;
2204 PyObject *exc = NULL;
2205
2206 /* Escaped strings will always be longer than the resulting
2207 Unicode string, so we start with size here and then reduce the
2208 length after conversion to the true value. (But decoding error
2209 handler might have to resize the string) */
2210 v = _PyUnicode_New(size);
2211 if (v == NULL)
2212 goto onError;
2213 if (size == 0)
2214 return (PyObject *)v;
2215 p = PyUnicode_AS_UNICODE(v);
2216 end = s + size;
2217 while (s < end) {
2218 unsigned char c;
2219 Py_UCS4 x;
2220 int i;
2221 int count;
2222
2223 /* Non-escape characters are interpreted as Unicode ordinals */
2224 if (*s != '\\') {
2225 *p++ = (unsigned char)*s++;
2226 continue;
2227 }
2228 startinpos = s-starts;
2229
2230 /* \u-escapes are only interpreted iff the number of leading
2231 backslashes if odd */
2232 bs = s;
2233 for (;s < end;) {
2234 if (*s != '\\')
2235 break;
2236 *p++ = (unsigned char)*s++;
2237 }
2238 if (((s - bs) & 1) == 0 ||
2239 s >= end ||
2240 (*s != 'u' && *s != 'U')) {
2241 continue;
2242 }
2243 p--;
2244 count = *s=='u' ? 4 : 8;
2245 s++;
2246
2247 /* \uXXXX with 4 hex digits, \Uxxxxxxxx with 8 */
2248 outpos = p-PyUnicode_AS_UNICODE(v);
2249 for (x = 0, i = 0; i < count; ++i, ++s) {
2250 c = (unsigned char)*s;
2251 if (!isxdigit(c)) {
2252 endinpos = s-starts;
2253 if (unicode_decode_call_errorhandler(
2254 errors, &errorHandler,
2255 "rawunicodeescape", "truncated \\uXXXX",
2256 starts, size, &startinpos, &endinpos, &exc, &s,
2257 (PyObject **)&v, &outpos, &p))
2258 goto onError;
2259 goto nextByte;
2260 }
2261 x = (x<<4) & ~0xF;
2262 if (c >= '0' && c <= '9')
2263 x += c - '0';
2264 else if (c >= 'a' && c <= 'f')
2265 x += 10 + c - 'a';
2266 else
2267 x += 10 + c - 'A';
2268 }
2269#ifndef Py_UNICODE_WIDE
2270 if (x > 0x10000) {
2271 if (unicode_decode_call_errorhandler(
2272 errors, &errorHandler,
2273 "rawunicodeescape", "\\Uxxxxxxxx out of range",
2274 starts, size, &startinpos, &endinpos, &exc, &s,
2275 (PyObject **)&v, &outpos, &p))
2276 goto onError;
2277 }
2278#endif
2279 *p++ = x;
2280 nextByte:
2281 ;
2282 }
2283 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
2284 goto onError;
2285 Py_XDECREF(errorHandler);
2286 Py_XDECREF(exc);
2287 return (PyObject *)v;
2288
2289 onError:
2290 Py_XDECREF(v);
2291 Py_XDECREF(errorHandler);
2292 Py_XDECREF(exc);
2293 return NULL;
2294}
2295
2296PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
2297 Py_ssize_t size)
2298{
2299 PyObject *repr;
2300 char *p;
2301 char *q;
2302
2303 static const char *hexdigit = "0123456789abcdef";
2304
2305#ifdef Py_UNICODE_WIDE
2306 repr = PyString_FromStringAndSize(NULL, 10 * size);
2307#else
2308 repr = PyString_FromStringAndSize(NULL, 6 * size);
2309#endif
2310 if (repr == NULL)
2311 return NULL;
2312 if (size == 0)
2313 return repr;
2314
2315 p = q = PyString_AS_STRING(repr);
2316 while (size-- > 0) {
2317 Py_UNICODE ch = *s++;
2318#ifdef Py_UNICODE_WIDE
2319 /* Map 32-bit characters to '\Uxxxxxxxx' */
2320 if (ch >= 0x10000) {
2321 *p++ = '\\';
2322 *p++ = 'U';
2323 *p++ = hexdigit[(ch >> 28) & 0xf];
2324 *p++ = hexdigit[(ch >> 24) & 0xf];
2325 *p++ = hexdigit[(ch >> 20) & 0xf];
2326 *p++ = hexdigit[(ch >> 16) & 0xf];
2327 *p++ = hexdigit[(ch >> 12) & 0xf];
2328 *p++ = hexdigit[(ch >> 8) & 0xf];
2329 *p++ = hexdigit[(ch >> 4) & 0xf];
2330 *p++ = hexdigit[ch & 15];
2331 }
2332 else
2333#endif
2334 /* Map 16-bit characters to '\uxxxx' */
2335 if (ch >= 256) {
2336 *p++ = '\\';
2337 *p++ = 'u';
2338 *p++ = hexdigit[(ch >> 12) & 0xf];
2339 *p++ = hexdigit[(ch >> 8) & 0xf];
2340 *p++ = hexdigit[(ch >> 4) & 0xf];
2341 *p++ = hexdigit[ch & 15];
2342 }
2343 /* Copy everything else as-is */
2344 else
2345 *p++ = (char) ch;
2346 }
2347 *p = '\0';
2348 _PyString_Resize(&repr, p - q);
2349 return repr;
2350}
2351
2352PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
2353{
2354 if (!PyUnicode_Check(unicode)) {
2355 PyErr_BadArgument();
2356 return NULL;
2357 }
2358 return PyUnicode_EncodeRawUnicodeEscape(PyUnicode_AS_UNICODE(unicode),
2359 PyUnicode_GET_SIZE(unicode));
2360}
2361
2362/* --- Unicode Internal Codec ------------------------------------------- */
2363
2364PyObject *_PyUnicode_DecodeUnicodeInternal(const char *s,
2365 Py_ssize_t size,
2366 const char *errors)
2367{
2368 const char *starts = s;
2369 Py_ssize_t startinpos;
2370 Py_ssize_t endinpos;
2371 Py_ssize_t outpos;
2372 PyUnicodeObject *v;
2373 Py_UNICODE *p;
2374 const char *end;
2375 const char *reason;
2376 PyObject *errorHandler = NULL;
2377 PyObject *exc = NULL;
2378
2379#ifdef Py_UNICODE_WIDE
2380 Py_UNICODE unimax = PyUnicode_GetMax();
2381#endif
2382
2383 /* XXX overflow detection missing */
2384 v = _PyUnicode_New((size+Py_UNICODE_SIZE-1)/ Py_UNICODE_SIZE);
2385 if (v == NULL)
2386 goto onError;
2387 if (PyUnicode_GetSize((PyObject *)v) == 0)
2388 return (PyObject *)v;
2389 p = PyUnicode_AS_UNICODE(v);
2390 end = s + size;
2391
2392 while (s < end) {
2393 memcpy(p, s, sizeof(Py_UNICODE));
2394 /* We have to sanity check the raw data, otherwise doom looms for
2395 some malformed UCS-4 data. */
2396 if (
2397 #ifdef Py_UNICODE_WIDE
2398 *p > unimax || *p < 0 ||
2399 #endif
2400 end-s < Py_UNICODE_SIZE
2401 )
2402 {
2403 startinpos = s - starts;
2404 if (end-s < Py_UNICODE_SIZE) {
2405 endinpos = end-starts;
2406 reason = "truncated input";
2407 }
2408 else {
2409 endinpos = s - starts + Py_UNICODE_SIZE;
2410 reason = "illegal code point (> 0x10FFFF)";
2411 }
2412 outpos = p - PyUnicode_AS_UNICODE(v);
2413 if (unicode_decode_call_errorhandler(
2414 errors, &errorHandler,
2415 "unicode_internal", reason,
2416 starts, size, &startinpos, &endinpos, &exc, &s,
2417 (PyObject **)&v, &outpos, &p)) {
2418 goto onError;
2419 }
2420 }
2421 else {
2422 p++;
2423 s += Py_UNICODE_SIZE;
2424 }
2425 }
2426
2427 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
2428 goto onError;
2429 Py_XDECREF(errorHandler);
2430 Py_XDECREF(exc);
2431 return (PyObject *)v;
2432
2433 onError:
2434 Py_XDECREF(v);
2435 Py_XDECREF(errorHandler);
2436 Py_XDECREF(exc);
2437 return NULL;
2438}
2439
2440/* --- Latin-1 Codec ------------------------------------------------------ */
2441
2442PyObject *PyUnicode_DecodeLatin1(const char *s,
2443 Py_ssize_t size,
2444 const char *errors)
2445{
2446 PyUnicodeObject *v;
2447 Py_UNICODE *p;
2448
2449 /* Latin-1 is equivalent to the first 256 ordinals in Unicode. */
2450 if (size == 1) {
2451 Py_UNICODE r = *(unsigned char*)s;
2452 return PyUnicode_FromUnicode(&r, 1);
2453 }
2454
2455 v = _PyUnicode_New(size);
2456 if (v == NULL)
2457 goto onError;
2458 if (size == 0)
2459 return (PyObject *)v;
2460 p = PyUnicode_AS_UNICODE(v);
2461 while (size-- > 0)
2462 *p++ = (unsigned char)*s++;
2463 return (PyObject *)v;
2464
2465 onError:
2466 Py_XDECREF(v);
2467 return NULL;
2468}
2469
2470/* create or adjust a UnicodeEncodeError */
2471static void make_encode_exception(PyObject **exceptionObject,
2472 const char *encoding,
2473 const Py_UNICODE *unicode, Py_ssize_t size,
2474 Py_ssize_t startpos, Py_ssize_t endpos,
2475 const char *reason)
2476{
2477 if (*exceptionObject == NULL) {
2478 *exceptionObject = PyUnicodeEncodeError_Create(
2479 encoding, unicode, size, startpos, endpos, reason);
2480 }
2481 else {
2482 if (PyUnicodeEncodeError_SetStart(*exceptionObject, startpos))
2483 goto onError;
2484 if (PyUnicodeEncodeError_SetEnd(*exceptionObject, endpos))
2485 goto onError;
2486 if (PyUnicodeEncodeError_SetReason(*exceptionObject, reason))
2487 goto onError;
2488 return;
2489 onError:
2490 Py_DECREF(*exceptionObject);
2491 *exceptionObject = NULL;
2492 }
2493}
2494
2495/* raises a UnicodeEncodeError */
2496static void raise_encode_exception(PyObject **exceptionObject,
2497 const char *encoding,
2498 const Py_UNICODE *unicode, Py_ssize_t size,
2499 Py_ssize_t startpos, Py_ssize_t endpos,
2500 const char *reason)
2501{
2502 make_encode_exception(exceptionObject,
2503 encoding, unicode, size, startpos, endpos, reason);
2504 if (*exceptionObject != NULL)
2505 PyCodec_StrictErrors(*exceptionObject);
2506}
2507
2508/* error handling callback helper:
2509 build arguments, call the callback and check the arguments,
2510 put the result into newpos and return the replacement string, which
2511 has to be freed by the caller */
2512static PyObject *unicode_encode_call_errorhandler(const char *errors,
2513 PyObject **errorHandler,
2514 const char *encoding, const char *reason,
2515 const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
2516 Py_ssize_t startpos, Py_ssize_t endpos,
2517 Py_ssize_t *newpos)
2518{
2519 static char *argparse = "O!n;encoding error handler must return (unicode, int) tuple";
2520
2521 PyObject *restuple;
2522 PyObject *resunicode;
2523
2524 if (*errorHandler == NULL) {
2525 *errorHandler = PyCodec_LookupError(errors);
2526 if (*errorHandler == NULL)
2527 return NULL;
2528 }
2529
2530 make_encode_exception(exceptionObject,
2531 encoding, unicode, size, startpos, endpos, reason);
2532 if (*exceptionObject == NULL)
2533 return NULL;
2534
2535 restuple = PyObject_CallFunctionObjArgs(
2536 *errorHandler, *exceptionObject, NULL);
2537 if (restuple == NULL)
2538 return NULL;
2539 if (!PyTuple_Check(restuple)) {
2540 PyErr_Format(PyExc_TypeError, &argparse[4]);
2541 Py_DECREF(restuple);
2542 return NULL;
2543 }
2544 if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type,
2545 &resunicode, newpos)) {
2546 Py_DECREF(restuple);
2547 return NULL;
2548 }
2549 if (*newpos<0)
2550 *newpos = size+*newpos;
2551 if (*newpos<0 || *newpos>size) {
2552 PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
2553 Py_DECREF(restuple);
2554 return NULL;
2555 }
2556 Py_INCREF(resunicode);
2557 Py_DECREF(restuple);
2558 return resunicode;
2559}
2560
2561static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
2562 Py_ssize_t size,
2563 const char *errors,
2564 int limit)
2565{
2566 /* output object */
2567 PyObject *res;
2568 /* pointers to the beginning and end+1 of input */
2569 const Py_UNICODE *startp = p;
2570 const Py_UNICODE *endp = p + size;
2571 /* pointer to the beginning of the unencodable characters */
2572 /* const Py_UNICODE *badp = NULL; */
2573 /* pointer into the output */
2574 char *str;
2575 /* current output position */
2576 Py_ssize_t respos = 0;
2577 Py_ssize_t ressize;
2578 const char *encoding = (limit == 256) ? "latin-1" : "ascii";
2579 const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
2580 PyObject *errorHandler = NULL;
2581 PyObject *exc = NULL;
2582 /* the following variable is used for caching string comparisons
2583 * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
2584 int known_errorHandler = -1;
2585
2586 /* allocate enough for a simple encoding without
2587 replacements, if we need more, we'll resize */
2588 res = PyString_FromStringAndSize(NULL, size);
2589 if (res == NULL)
2590 goto onError;
2591 if (size == 0)
2592 return res;
2593 str = PyString_AS_STRING(res);
2594 ressize = size;
2595
2596 while (p<endp) {
2597 Py_UNICODE c = *p;
2598
2599 /* can we encode this? */
2600 if (c<limit) {
2601 /* no overflow check, because we know that the space is enough */
2602 *str++ = (char)c;
2603 ++p;
2604 }
2605 else {
2606 Py_ssize_t unicodepos = p-startp;
2607 Py_ssize_t requiredsize;
2608 PyObject *repunicode;
2609 Py_ssize_t repsize;
2610 Py_ssize_t newpos;
2611 Py_ssize_t respos;
2612 Py_UNICODE *uni2;
2613 /* startpos for collecting unencodable chars */
2614 const Py_UNICODE *collstart = p;
2615 const Py_UNICODE *collend = p;
2616 /* find all unecodable characters */
2617 while ((collend < endp) && ((*collend)>=limit))
2618 ++collend;
2619 /* cache callback name lookup (if not done yet, i.e. it's the first error) */
2620 if (known_errorHandler==-1) {
2621 if ((errors==NULL) || (!strcmp(errors, "strict")))
2622 known_errorHandler = 1;
2623 else if (!strcmp(errors, "replace"))
2624 known_errorHandler = 2;
2625 else if (!strcmp(errors, "ignore"))
2626 known_errorHandler = 3;
2627 else if (!strcmp(errors, "xmlcharrefreplace"))
2628 known_errorHandler = 4;
2629 else
2630 known_errorHandler = 0;
2631 }
2632 switch (known_errorHandler) {
2633 case 1: /* strict */
2634 raise_encode_exception(&exc, encoding, startp, size, collstart-startp, collend-startp, reason);
2635 goto onError;
2636 case 2: /* replace */
2637 while (collstart++<collend)
2638 *str++ = '?'; /* fall through */
2639 case 3: /* ignore */
2640 p = collend;
2641 break;
2642 case 4: /* xmlcharrefreplace */
2643 respos = str-PyString_AS_STRING(res);
2644 /* determine replacement size (temporarily (mis)uses p) */
2645 for (p = collstart, repsize = 0; p < collend; ++p) {
2646 if (*p<10)
2647 repsize += 2+1+1;
2648 else if (*p<100)
2649 repsize += 2+2+1;
2650 else if (*p<1000)
2651 repsize += 2+3+1;
2652 else if (*p<10000)
2653 repsize += 2+4+1;
2654#ifndef Py_UNICODE_WIDE
2655 else
2656 repsize += 2+5+1;
2657#else
2658 else if (*p<100000)
2659 repsize += 2+5+1;
2660 else if (*p<1000000)
2661 repsize += 2+6+1;
2662 else
2663 repsize += 2+7+1;
2664#endif
2665 }
2666 requiredsize = respos+repsize+(endp-collend);
2667 if (requiredsize > ressize) {
2668 if (requiredsize<2*ressize)
2669 requiredsize = 2*ressize;
2670 if (_PyString_Resize(&res, requiredsize))
2671 goto onError;
2672 str = PyString_AS_STRING(res) + respos;
2673 ressize = requiredsize;
2674 }
2675 /* generate replacement (temporarily (mis)uses p) */
2676 for (p = collstart; p < collend; ++p) {
2677 str += sprintf(str, "&#%d;", (int)*p);
2678 }
2679 p = collend;
2680 break;
2681 default:
2682 repunicode = unicode_encode_call_errorhandler(errors, &errorHandler,
2683 encoding, reason, startp, size, &exc,
2684 collstart-startp, collend-startp, &newpos);
2685 if (repunicode == NULL)
2686 goto onError;
2687 /* need more space? (at least enough for what we
2688 have+the replacement+the rest of the string, so
2689 we won't have to check space for encodable characters) */
2690 respos = str-PyString_AS_STRING(res);
2691 repsize = PyUnicode_GET_SIZE(repunicode);
2692 requiredsize = respos+repsize+(endp-collend);
2693 if (requiredsize > ressize) {
2694 if (requiredsize<2*ressize)
2695 requiredsize = 2*ressize;
2696 if (_PyString_Resize(&res, requiredsize)) {
2697 Py_DECREF(repunicode);
2698 goto onError;
2699 }
2700 str = PyString_AS_STRING(res) + respos;
2701 ressize = requiredsize;
2702 }
2703 /* check if there is anything unencodable in the replacement
2704 and copy it to the output */
2705 for (uni2 = PyUnicode_AS_UNICODE(repunicode);repsize-->0; ++uni2, ++str) {
2706 c = *uni2;
2707 if (c >= limit) {
2708 raise_encode_exception(&exc, encoding, startp, size,
2709 unicodepos, unicodepos+1, reason);
2710 Py_DECREF(repunicode);
2711 goto onError;
2712 }
2713 *str = (char)c;
2714 }
2715 p = startp + newpos;
2716 Py_DECREF(repunicode);
2717 }
2718 }
2719 }
2720 /* Resize if we allocated to much */
2721 respos = str-PyString_AS_STRING(res);
2722 if (respos<ressize)
2723 /* If this falls res will be NULL */
2724 _PyString_Resize(&res, respos);
2725 Py_XDECREF(errorHandler);
2726 Py_XDECREF(exc);
2727 return res;
2728
2729 onError:
2730 Py_XDECREF(res);
2731 Py_XDECREF(errorHandler);
2732 Py_XDECREF(exc);
2733 return NULL;
2734}
2735
2736PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p,
2737 Py_ssize_t size,
2738 const char *errors)
2739{
2740 return unicode_encode_ucs1(p, size, errors, 256);
2741}
2742
2743PyObject *PyUnicode_AsLatin1String(PyObject *unicode)
2744{
2745 if (!PyUnicode_Check(unicode)) {
2746 PyErr_BadArgument();
2747 return NULL;
2748 }
2749 return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(unicode),
2750 PyUnicode_GET_SIZE(unicode),
2751 NULL);
2752}
2753
2754/* --- 7-bit ASCII Codec -------------------------------------------------- */
2755
2756PyObject *PyUnicode_DecodeASCII(const char *s,
2757 Py_ssize_t size,
2758 const char *errors)
2759{
2760 const char *starts = s;
2761 PyUnicodeObject *v;
2762 Py_UNICODE *p;
2763 Py_ssize_t startinpos;
2764 Py_ssize_t endinpos;
2765 Py_ssize_t outpos;
2766 const char *e;
2767 PyObject *errorHandler = NULL;
2768 PyObject *exc = NULL;
2769
2770 /* ASCII is equivalent to the first 128 ordinals in Unicode. */
2771 if (size == 1 && *(unsigned char*)s < 128) {
2772 Py_UNICODE r = *(unsigned char*)s;
2773 return PyUnicode_FromUnicode(&r, 1);
2774 }
2775
2776 v = _PyUnicode_New(size);
2777 if (v == NULL)
2778 goto onError;
2779 if (size == 0)
2780 return (PyObject *)v;
2781 p = PyUnicode_AS_UNICODE(v);
2782 e = s + size;
2783 while (s < e) {
2784 register unsigned char c = (unsigned char)*s;
2785 if (c < 128) {
2786 *p++ = c;
2787 ++s;
2788 }
2789 else {
2790 startinpos = s-starts;
2791 endinpos = startinpos + 1;
2792 outpos = p - (Py_UNICODE *)PyUnicode_AS_UNICODE(v);
2793 if (unicode_decode_call_errorhandler(
2794 errors, &errorHandler,
2795 "ascii", "ordinal not in range(128)",
2796 starts, size, &startinpos, &endinpos, &exc, &s,
2797 (PyObject **)&v, &outpos, &p))
2798 goto onError;
2799 }
2800 }
2801 if (p - PyUnicode_AS_UNICODE(v) < PyString_GET_SIZE(v))
2802 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
2803 goto onError;
2804 Py_XDECREF(errorHandler);
2805 Py_XDECREF(exc);
2806 return (PyObject *)v;
2807
2808 onError:
2809 Py_XDECREF(v);
2810 Py_XDECREF(errorHandler);
2811 Py_XDECREF(exc);
2812 return NULL;
2813}
2814
2815PyObject *PyUnicode_EncodeASCII(const Py_UNICODE *p,
2816 Py_ssize_t size,
2817 const char *errors)
2818{
2819 return unicode_encode_ucs1(p, size, errors, 128);
2820}
2821
2822PyObject *PyUnicode_AsASCIIString(PyObject *unicode)
2823{
2824 if (!PyUnicode_Check(unicode)) {
2825 PyErr_BadArgument();
2826 return NULL;
2827 }
2828 return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(unicode),
2829 PyUnicode_GET_SIZE(unicode),
2830 NULL);
2831}
2832
2833#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T)
2834
2835/* --- MBCS codecs for Windows -------------------------------------------- */
2836
2837#if SIZEOF_INT < SIZEOF_SSIZE_T
2838#define NEED_RETRY
2839#endif
2840
2841/* XXX This code is limited to "true" double-byte encodings, as
2842 a) it assumes an incomplete character consists of a single byte, and
2843 b) IsDBCSLeadByte (probably) does not work for non-DBCS multi-byte
2844 encodings, see IsDBCSLeadByteEx documentation. */
2845
2846static int is_dbcs_lead_byte(const char *s, int offset)
2847{
2848 const char *curr = s + offset;
2849
2850 if (IsDBCSLeadByte(*curr)) {
2851 const char *prev = CharPrev(s, curr);
2852 return (prev == curr) || !IsDBCSLeadByte(*prev) || (curr - prev == 2);
2853 }
2854 return 0;
2855}
2856
2857/*
2858 * Decode MBCS string into unicode object. If 'final' is set, converts
2859 * trailing lead-byte too. Returns consumed size if succeed, -1 otherwise.
2860 */
2861static int decode_mbcs(PyUnicodeObject **v,
2862 const char *s, /* MBCS string */
2863 int size, /* sizeof MBCS string */
2864 int final)
2865{
2866 Py_UNICODE *p;
2867 Py_ssize_t n = 0;
2868 int usize = 0;
2869
2870 assert(size >= 0);
2871
2872 /* Skip trailing lead-byte unless 'final' is set */
2873 if (!final && size >= 1 && is_dbcs_lead_byte(s, size - 1))
2874 --size;
2875
2876 /* First get the size of the result */
2877 if (size > 0) {
2878 usize = MultiByteToWideChar(CP_ACP, 0, s, size, NULL, 0);
2879 if (usize == 0) {
2880 PyErr_SetFromWindowsErrWithFilename(0, NULL);
2881 return -1;
2882 }
2883 }
2884
2885 if (*v == NULL) {
2886 /* Create unicode object */
2887 *v = _PyUnicode_New(usize);
2888 if (*v == NULL)
2889 return -1;
2890 }
2891 else {
2892 /* Extend unicode object */
2893 n = PyUnicode_GET_SIZE(*v);
2894 if (_PyUnicode_Resize(v, n + usize) < 0)
2895 return -1;
2896 }
2897
2898 /* Do the conversion */
2899 if (size > 0) {
2900 p = PyUnicode_AS_UNICODE(*v) + n;
2901 if (0 == MultiByteToWideChar(CP_ACP, 0, s, size, p, usize)) {
2902 PyErr_SetFromWindowsErrWithFilename(0, NULL);
2903 return -1;
2904 }
2905 }
2906
2907 return size;
2908}
2909
2910PyObject *PyUnicode_DecodeMBCSStateful(const char *s,
2911 Py_ssize_t size,
2912 const char *errors,
2913 Py_ssize_t *consumed)
2914{
2915 PyUnicodeObject *v = NULL;
2916 int done;
2917
2918 if (consumed)
2919 *consumed = 0;
2920
2921#ifdef NEED_RETRY
2922 retry:
2923 if (size > INT_MAX)
2924 done = decode_mbcs(&v, s, INT_MAX, 0);
2925 else
2926#endif
2927 done = decode_mbcs(&v, s, (int)size, !consumed);
2928
2929 if (done < 0) {
2930 Py_XDECREF(v);
2931 return NULL;
2932 }
2933
2934 if (consumed)
2935 *consumed += done;
2936
2937#ifdef NEED_RETRY
2938 if (size > INT_MAX) {
2939 s += done;
2940 size -= done;
2941 goto retry;
2942 }
2943#endif
2944
2945 return (PyObject *)v;
2946}
2947
2948PyObject *PyUnicode_DecodeMBCS(const char *s,
2949 Py_ssize_t size,
2950 const char *errors)
2951{
2952 return PyUnicode_DecodeMBCSStateful(s, size, errors, NULL);
2953}
2954
2955/*
2956 * Convert unicode into string object (MBCS).
2957 * Returns 0 if succeed, -1 otherwise.
2958 */
2959static int encode_mbcs(PyObject **repr,
2960 const Py_UNICODE *p, /* unicode */
2961 int size) /* size of unicode */
2962{
2963 int mbcssize = 0;
2964 Py_ssize_t n = 0;
2965
2966 assert(size >= 0);
2967
2968 /* First get the size of the result */
2969 if (size > 0) {
2970 mbcssize = WideCharToMultiByte(CP_ACP, 0, p, size, NULL, 0, NULL, NULL);
2971 if (mbcssize == 0) {
2972 PyErr_SetFromWindowsErrWithFilename(0, NULL);
2973 return -1;
2974 }
2975 }
2976
2977 if (*repr == NULL) {
2978 /* Create string object */
2979 *repr = PyString_FromStringAndSize(NULL, mbcssize);
2980 if (*repr == NULL)
2981 return -1;
2982 }
2983 else {
2984 /* Extend string object */
2985 n = PyString_Size(*repr);
2986 if (_PyString_Resize(repr, n + mbcssize) < 0)
2987 return -1;
2988 }
2989
2990 /* Do the conversion */
2991 if (size > 0) {
2992 char *s = PyString_AS_STRING(*repr) + n;
2993 if (0 == WideCharToMultiByte(CP_ACP, 0, p, size, s, mbcssize, NULL, NULL)) {
2994 PyErr_SetFromWindowsErrWithFilename(0, NULL);
2995 return -1;
2996 }
2997 }
2998
2999 return 0;
3000}
3001
3002PyObject *PyUnicode_EncodeMBCS(const Py_UNICODE *p,
3003 Py_ssize_t size,
3004 const char *errors)
3005{
3006 PyObject *repr = NULL;
3007 int ret;
3008
3009#ifdef NEED_RETRY
3010 retry:
3011 if (size > INT_MAX)
3012 ret = encode_mbcs(&repr, p, INT_MAX);
3013 else
3014#endif
3015 ret = encode_mbcs(&repr, p, (int)size);
3016
3017 if (ret < 0) {
3018 Py_XDECREF(repr);
3019 return NULL;
3020 }
3021
3022#ifdef NEED_RETRY
3023 if (size > INT_MAX) {
3024 p += INT_MAX;
3025 size -= INT_MAX;
3026 goto retry;
3027 }
3028#endif
3029
3030 return repr;
3031}
3032
3033PyObject *PyUnicode_AsMBCSString(PyObject *unicode)
3034{
3035 if (!PyUnicode_Check(unicode)) {
3036 PyErr_BadArgument();
3037 return NULL;
3038 }
3039 return PyUnicode_EncodeMBCS(PyUnicode_AS_UNICODE(unicode),
3040 PyUnicode_GET_SIZE(unicode),
3041 NULL);
3042}
3043
3044#undef NEED_RETRY
3045
3046#endif /* MS_WINDOWS */
3047
3048/* --- Character Mapping Codec -------------------------------------------- */
3049
3050PyObject *PyUnicode_DecodeCharmap(const char *s,
3051 Py_ssize_t size,
3052 PyObject *mapping,
3053 const char *errors)
3054{
3055 const char *starts = s;
3056 Py_ssize_t startinpos;
3057 Py_ssize_t endinpos;
3058 Py_ssize_t outpos;
3059 const char *e;
3060 PyUnicodeObject *v;
3061 Py_UNICODE *p;
3062 Py_ssize_t extrachars = 0;
3063 PyObject *errorHandler = NULL;
3064 PyObject *exc = NULL;
3065 Py_UNICODE *mapstring = NULL;
3066 Py_ssize_t maplen = 0;
3067
3068 /* Default to Latin-1 */
3069 if (mapping == NULL)
3070 return PyUnicode_DecodeLatin1(s, size, errors);
3071
3072 v = _PyUnicode_New(size);
3073 if (v == NULL)
3074 goto onError;
3075 if (size == 0)
3076 return (PyObject *)v;
3077 p = PyUnicode_AS_UNICODE(v);
3078 e = s + size;
3079 if (PyUnicode_CheckExact(mapping)) {
3080 mapstring = PyUnicode_AS_UNICODE(mapping);
3081 maplen = PyUnicode_GET_SIZE(mapping);
3082 while (s < e) {
3083 unsigned char ch = *s;
3084 Py_UNICODE x = 0xfffe; /* illegal value */
3085
3086 if (ch < maplen)
3087 x = mapstring[ch];
3088
3089 if (x == 0xfffe) {
3090 /* undefined mapping */
3091 outpos = p-PyUnicode_AS_UNICODE(v);
3092 startinpos = s-starts;
3093 endinpos = startinpos+1;
3094 if (unicode_decode_call_errorhandler(
3095 errors, &errorHandler,
3096 "charmap", "character maps to <undefined>",
3097 starts, size, &startinpos, &endinpos, &exc, &s,
3098 (PyObject **)&v, &outpos, &p)) {
3099 goto onError;
3100 }
3101 continue;
3102 }
3103 *p++ = x;
3104 ++s;
3105 }
3106 }
3107 else {
3108 while (s < e) {
3109 unsigned char ch = *s;
3110 PyObject *w, *x;
3111
3112 /* Get mapping (char ordinal -> integer, Unicode char or None) */
3113 w = PyInt_FromLong((long)ch);
3114 if (w == NULL)
3115 goto onError;
3116 x = PyObject_GetItem(mapping, w);
3117 Py_DECREF(w);
3118 if (x == NULL) {
3119 if (PyErr_ExceptionMatches(PyExc_LookupError)) {
3120 /* No mapping found means: mapping is undefined. */
3121 PyErr_Clear();
3122 x = Py_None;
3123 Py_INCREF(x);
3124 } else
3125 goto onError;
3126 }
3127
3128 /* Apply mapping */
3129 if (PyInt_Check(x)) {
3130 long value = PyInt_AS_LONG(x);
3131 if (value < 0 || value > 65535) {
3132 PyErr_SetString(PyExc_TypeError,
3133 "character mapping must be in range(65536)");
3134 Py_DECREF(x);
3135 goto onError;
3136 }
3137 *p++ = (Py_UNICODE)value;
3138 }
3139 else if (x == Py_None) {
3140 /* undefined mapping */
3141 outpos = p-PyUnicode_AS_UNICODE(v);
3142 startinpos = s-starts;
3143 endinpos = startinpos+1;
3144 if (unicode_decode_call_errorhandler(
3145 errors, &errorHandler,
3146 "charmap", "character maps to <undefined>",
3147 starts, size, &startinpos, &endinpos, &exc, &s,
3148 (PyObject **)&v, &outpos, &p)) {
3149 Py_DECREF(x);
3150 goto onError;
3151 }
3152 Py_DECREF(x);
3153 continue;
3154 }
3155 else if (PyUnicode_Check(x)) {
3156 Py_ssize_t targetsize = PyUnicode_GET_SIZE(x);
3157
3158 if (targetsize == 1)
3159 /* 1-1 mapping */
3160 *p++ = *PyUnicode_AS_UNICODE(x);
3161
3162 else if (targetsize > 1) {
3163 /* 1-n mapping */
3164 if (targetsize > extrachars) {
3165 /* resize first */
3166 Py_ssize_t oldpos = p - PyUnicode_AS_UNICODE(v);
3167 Py_ssize_t needed = (targetsize - extrachars) + \
3168 (targetsize << 2);
3169 extrachars += needed;
3170 /* XXX overflow detection missing */
3171 if (_PyUnicode_Resize(&v,
3172 PyUnicode_GET_SIZE(v) + needed) < 0) {
3173 Py_DECREF(x);
3174 goto onError;
3175 }
3176 p = PyUnicode_AS_UNICODE(v) + oldpos;
3177 }
3178 Py_UNICODE_COPY(p,
3179 PyUnicode_AS_UNICODE(x),
3180 targetsize);
3181 p += targetsize;
3182 extrachars -= targetsize;
3183 }
3184 /* 1-0 mapping: skip the character */
3185 }
3186 else {
3187 /* wrong return value */
3188 PyErr_SetString(PyExc_TypeError,
3189 "character mapping must return integer, None or unicode");
3190 Py_DECREF(x);
3191 goto onError;
3192 }
3193 Py_DECREF(x);
3194 ++s;
3195 }
3196 }
3197 if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v))
3198 if (_PyUnicode_Resize(&v, p - PyUnicode_AS_UNICODE(v)) < 0)
3199 goto onError;
3200 Py_XDECREF(errorHandler);
3201 Py_XDECREF(exc);
3202 return (PyObject *)v;
3203
3204 onError:
3205 Py_XDECREF(errorHandler);
3206 Py_XDECREF(exc);
3207 Py_XDECREF(v);
3208 return NULL;
3209}
3210
3211/* Charmap encoding: the lookup table */
3212
3213struct encoding_map{
3214 PyObject_HEAD
3215 unsigned char level1[32];
3216 int count2, count3;
3217 unsigned char level23[1];
3218};
3219
3220static PyObject*
3221encoding_map_size(PyObject *obj, PyObject* args)
3222{
3223 struct encoding_map *map = (struct encoding_map*)obj;
3224 return PyInt_FromLong(sizeof(*map) - 1 + 16*map->count2 +
3225 128*map->count3);
3226}
3227
3228static PyMethodDef encoding_map_methods[] = {
3229 {"size", encoding_map_size, METH_NOARGS,
3230 PyDoc_STR("Return the size (in bytes) of this object") },
3231 { 0 }
3232};
3233
3234static void
3235encoding_map_dealloc(PyObject* o)
3236{
3237 PyObject_FREE(o);
3238}
3239
3240static PyTypeObject EncodingMapType = {
3241 PyObject_HEAD_INIT(NULL)
3242 0, /*ob_size*/
3243 "EncodingMap", /*tp_name*/
3244 sizeof(struct encoding_map), /*tp_basicsize*/
3245 0, /*tp_itemsize*/
3246 /* methods */
3247 encoding_map_dealloc, /*tp_dealloc*/
3248 0, /*tp_print*/
3249 0, /*tp_getattr*/
3250 0, /*tp_setattr*/
3251 0, /*tp_compare*/
3252 0, /*tp_repr*/
3253 0, /*tp_as_number*/
3254 0, /*tp_as_sequence*/
3255 0, /*tp_as_mapping*/
3256 0, /*tp_hash*/
3257 0, /*tp_call*/
3258 0, /*tp_str*/
3259 0, /*tp_getattro*/
3260 0, /*tp_setattro*/
3261 0, /*tp_as_buffer*/
3262 Py_TPFLAGS_DEFAULT, /*tp_flags*/
3263 0, /*tp_doc*/
3264 0, /*tp_traverse*/
3265 0, /*tp_clear*/
3266 0, /*tp_richcompare*/
3267 0, /*tp_weaklistoffset*/
3268 0, /*tp_iter*/
3269 0, /*tp_iternext*/
3270 encoding_map_methods, /*tp_methods*/
3271 0, /*tp_members*/
3272 0, /*tp_getset*/
3273 0, /*tp_base*/
3274 0, /*tp_dict*/
3275 0, /*tp_descr_get*/
3276 0, /*tp_descr_set*/
3277 0, /*tp_dictoffset*/
3278 0, /*tp_init*/
3279 0, /*tp_alloc*/
3280 0, /*tp_new*/
3281 0, /*tp_free*/
3282 0, /*tp_is_gc*/
3283};
3284
3285PyObject*
3286PyUnicode_BuildEncodingMap(PyObject* string)
3287{
3288 Py_UNICODE *decode;
3289 PyObject *result;
3290 struct encoding_map *mresult;
3291 int i;
3292 int need_dict = 0;
3293 unsigned char level1[32];
3294 unsigned char level2[512];
3295 unsigned char *mlevel1, *mlevel2, *mlevel3;
3296 int count2 = 0, count3 = 0;
3297
3298 if (!PyUnicode_Check(string) || PyUnicode_GetSize(string) != 256) {
3299 PyErr_BadArgument();
3300 return NULL;
3301 }
3302 decode = PyUnicode_AS_UNICODE(string);
3303 memset(level1, 0xFF, sizeof level1);
3304 memset(level2, 0xFF, sizeof level2);
3305
3306 /* If there isn't a one-to-one mapping of NULL to \0,
3307 or if there are non-BMP characters, we need to use
3308 a mapping dictionary. */
3309 if (decode[0] != 0)
3310 need_dict = 1;
3311 for (i = 1; i < 256; i++) {
3312 int l1, l2;
3313 if (decode[i] == 0
3314 #ifdef Py_UNICODE_WIDE
3315 || decode[i] > 0xFFFF
3316 #endif
3317 ) {
3318 need_dict = 1;
3319 break;
3320 }
3321 if (decode[i] == 0xFFFE)
3322 /* unmapped character */
3323 continue;
3324 l1 = decode[i] >> 11;
3325 l2 = decode[i] >> 7;
3326 if (level1[l1] == 0xFF)
3327 level1[l1] = count2++;
3328 if (level2[l2] == 0xFF)
3329 level2[l2] = count3++;
3330 }
3331
3332 if (count2 >= 0xFF || count3 >= 0xFF)
3333 need_dict = 1;
3334
3335 if (need_dict) {
3336 PyObject *result = PyDict_New();
3337 PyObject *key, *value;
3338 if (!result)
3339 return NULL;
3340 for (i = 0; i < 256; i++) {
3341 key = value = NULL;
3342 key = PyInt_FromLong(decode[i]);
3343 value = PyInt_FromLong(i);
3344 if (!key || !value)
3345 goto failed1;
3346 if (PyDict_SetItem(result, key, value) == -1)
3347 goto failed1;
3348 Py_DECREF(key);
3349 Py_DECREF(value);
3350 }
3351 return result;
3352 failed1:
3353 Py_XDECREF(key);
3354 Py_XDECREF(value);
3355 Py_DECREF(result);
3356 return NULL;
3357 }
3358
3359 /* Create a three-level trie */
3360 result = PyObject_MALLOC(sizeof(struct encoding_map) +
3361 16*count2 + 128*count3 - 1);
3362 if (!result)
3363 return PyErr_NoMemory();
3364 PyObject_Init(result, &EncodingMapType);
3365 mresult = (struct encoding_map*)result;
3366 mresult->count2 = count2;
3367 mresult->count3 = count3;
3368 mlevel1 = mresult->level1;
3369 mlevel2 = mresult->level23;
3370 mlevel3 = mresult->level23 + 16*count2;
3371 memcpy(mlevel1, level1, 32);
3372 memset(mlevel2, 0xFF, 16*count2);
3373 memset(mlevel3, 0, 128*count3);
3374 count3 = 0;
3375 for (i = 1; i < 256; i++) {
3376 int o1, o2, o3, i2, i3;
3377 if (decode[i] == 0xFFFE)
3378 /* unmapped character */
3379 continue;
3380 o1 = decode[i]>>11;
3381 o2 = (decode[i]>>7) & 0xF;
3382 i2 = 16*mlevel1[o1] + o2;
3383 if (mlevel2[i2] == 0xFF)
3384 mlevel2[i2] = count3++;
3385 o3 = decode[i] & 0x7F;
3386 i3 = 128*mlevel2[i2] + o3;
3387 mlevel3[i3] = i;
3388 }
3389 return result;
3390}
3391
3392static int
3393encoding_map_lookup(Py_UNICODE c, PyObject *mapping)
3394{
3395 struct encoding_map *map = (struct encoding_map*)mapping;
3396 int l1 = c>>11;
3397 int l2 = (c>>7) & 0xF;
3398 int l3 = c & 0x7F;
3399 int i;
3400
3401#ifdef Py_UNICODE_WIDE
3402 if (c > 0xFFFF) {
3403 return -1;
3404 }
3405#endif
3406 if (c == 0)
3407 return 0;
3408 /* level 1*/
3409 i = map->level1[l1];
3410 if (i == 0xFF) {
3411 return -1;
3412 }
3413 /* level 2*/
3414 i = map->level23[16*i+l2];
3415 if (i == 0xFF) {
3416 return -1;
3417 }
3418 /* level 3 */
3419 i = map->level23[16*map->count2 + 128*i + l3];
3420 if (i == 0) {
3421 return -1;
3422 }
3423 return i;
3424}
3425
3426/* Lookup the character ch in the mapping. If the character
3427 can't be found, Py_None is returned (or NULL, if another
3428 error occurred). */
3429static PyObject *charmapencode_lookup(Py_UNICODE c, PyObject *mapping)
3430{
3431 PyObject *w = PyInt_FromLong((long)c);
3432 PyObject *x;
3433
3434 if (w == NULL)
3435 return NULL;
3436 x = PyObject_GetItem(mapping, w);
3437 Py_DECREF(w);
3438 if (x == NULL) {
3439 if (PyErr_ExceptionMatches(PyExc_LookupError)) {
3440 /* No mapping found means: mapping is undefined. */
3441 PyErr_Clear();
3442 x = Py_None;
3443 Py_INCREF(x);
3444 return x;
3445 } else
3446 return NULL;
3447 }
3448 else if (x == Py_None)
3449 return x;
3450 else if (PyInt_Check(x)) {
3451 long value = PyInt_AS_LONG(x);
3452 if (value < 0 || value > 255) {
3453 PyErr_SetString(PyExc_TypeError,
3454 "character mapping must be in range(256)");
3455 Py_DECREF(x);
3456 return NULL;
3457 }
3458 return x;
3459 }
3460 else if (PyString_Check(x))
3461 return x;
3462 else {
3463 /* wrong return value */
3464 PyErr_SetString(PyExc_TypeError,
3465 "character mapping must return integer, None or str");
3466 Py_DECREF(x);
3467 return NULL;
3468 }
3469}
3470
3471static int
3472charmapencode_resize(PyObject **outobj, Py_ssize_t *outpos, Py_ssize_t requiredsize)
3473{
3474 Py_ssize_t outsize = PyString_GET_SIZE(*outobj);
3475 /* exponentially overallocate to minimize reallocations */
3476 if (requiredsize < 2*outsize)
3477 requiredsize = 2*outsize;
3478 if (_PyString_Resize(outobj, requiredsize)) {
3479 return 0;
3480 }
3481 return 1;
3482}
3483
3484typedef enum charmapencode_result {
3485 enc_SUCCESS, enc_FAILED, enc_EXCEPTION
3486}charmapencode_result;
3487/* lookup the character, put the result in the output string and adjust
3488 various state variables. Reallocate the output string if not enough
3489 space is available. Return a new reference to the object that
3490 was put in the output buffer, or Py_None, if the mapping was undefined
3491 (in which case no character was written) or NULL, if a
3492 reallocation error occurred. The caller must decref the result */
3493static
3494charmapencode_result charmapencode_output(Py_UNICODE c, PyObject *mapping,
3495 PyObject **outobj, Py_ssize_t *outpos)
3496{
3497 PyObject *rep;
3498 char *outstart;
3499 Py_ssize_t outsize = PyString_GET_SIZE(*outobj);
3500
3501 if (mapping->ob_type == &EncodingMapType) {
3502 int res = encoding_map_lookup(c, mapping);
3503 Py_ssize_t requiredsize = *outpos+1;
3504 if (res == -1)
3505 return enc_FAILED;
3506 if (outsize<requiredsize)
3507 if (!charmapencode_resize(outobj, outpos, requiredsize))
3508 return enc_EXCEPTION;
3509 outstart = PyString_AS_STRING(*outobj);
3510 outstart[(*outpos)++] = (char)res;
3511 return enc_SUCCESS;
3512 }
3513
3514 rep = charmapencode_lookup(c, mapping);
3515 if (rep==NULL)
3516 return enc_EXCEPTION;
3517 else if (rep==Py_None) {
3518 Py_DECREF(rep);
3519 return enc_FAILED;
3520 } else {
3521 if (PyInt_Check(rep)) {
3522 Py_ssize_t requiredsize = *outpos+1;
3523 if (outsize<requiredsize)
3524 if (!charmapencode_resize(outobj, outpos, requiredsize)) {
3525 Py_DECREF(rep);
3526 return enc_EXCEPTION;
3527 }
3528 outstart = PyString_AS_STRING(*outobj);
3529 outstart[(*outpos)++] = (char)PyInt_AS_LONG(rep);
3530 }
3531 else {
3532 const char *repchars = PyString_AS_STRING(rep);
3533 Py_ssize_t repsize = PyString_GET_SIZE(rep);
3534 Py_ssize_t requiredsize = *outpos+repsize;
3535 if (outsize<requiredsize)
3536 if (!charmapencode_resize(outobj, outpos, requiredsize)) {
3537 Py_DECREF(rep);
3538 return enc_EXCEPTION;
3539 }
3540 outstart = PyString_AS_STRING(*outobj);
3541 memcpy(outstart + *outpos, repchars, repsize);
3542 *outpos += repsize;
3543 }
3544 }
3545 Py_DECREF(rep);
3546 return enc_SUCCESS;
3547}
3548
3549/* handle an error in PyUnicode_EncodeCharmap
3550 Return 0 on success, -1 on error */
3551static
3552int charmap_encoding_error(
3553 const Py_UNICODE *p, Py_ssize_t size, Py_ssize_t *inpos, PyObject *mapping,
3554 PyObject **exceptionObject,
3555 int *known_errorHandler, PyObject **errorHandler, const char *errors,
3556 PyObject **res, Py_ssize_t *respos)
3557{
3558 PyObject *repunicode = NULL; /* initialize to prevent gcc warning */
3559 Py_ssize_t repsize;
3560 Py_ssize_t newpos;
3561 Py_UNICODE *uni2;
3562 /* startpos for collecting unencodable chars */
3563 Py_ssize_t collstartpos = *inpos;
3564 Py_ssize_t collendpos = *inpos+1;
3565 Py_ssize_t collpos;
3566 char *encoding = "charmap";
3567 char *reason = "character maps to <undefined>";
3568 charmapencode_result x;
3569
3570 /* find all unencodable characters */
3571 while (collendpos < size) {
3572 PyObject *rep;
3573 if (mapping->ob_type == &EncodingMapType) {
3574 int res = encoding_map_lookup(p[collendpos], mapping);
3575 if (res != -1)
3576 break;
3577 ++collendpos;
3578 continue;
3579 }
3580
3581 rep = charmapencode_lookup(p[collendpos], mapping);
3582 if (rep==NULL)
3583 return -1;
3584 else if (rep!=Py_None) {
3585 Py_DECREF(rep);
3586 break;
3587 }
3588 Py_DECREF(rep);
3589 ++collendpos;
3590 }
3591 /* cache callback name lookup
3592 * (if not done yet, i.e. it's the first error) */
3593 if (*known_errorHandler==-1) {
3594 if ((errors==NULL) || (!strcmp(errors, "strict")))
3595 *known_errorHandler = 1;
3596 else if (!strcmp(errors, "replace"))
3597 *known_errorHandler = 2;
3598 else if (!strcmp(errors, "ignore"))
3599 *known_errorHandler = 3;
3600 else if (!strcmp(errors, "xmlcharrefreplace"))
3601 *known_errorHandler = 4;
3602 else
3603 *known_errorHandler = 0;
3604 }
3605 switch (*known_errorHandler) {
3606 case 1: /* strict */
3607 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3608 return -1;
3609 case 2: /* replace */
3610 for (collpos = collstartpos; collpos<collendpos; ++collpos) {
3611 x = charmapencode_output('?', mapping, res, respos);
3612 if (x==enc_EXCEPTION) {
3613 return -1;
3614 }
3615 else if (x==enc_FAILED) {
3616 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3617 return -1;
3618 }
3619 }
3620 /* fall through */
3621 case 3: /* ignore */
3622 *inpos = collendpos;
3623 break;
3624 case 4: /* xmlcharrefreplace */
3625 /* generate replacement (temporarily (mis)uses p) */
3626 for (collpos = collstartpos; collpos < collendpos; ++collpos) {
3627 char buffer[2+29+1+1];
3628 char *cp;
3629 sprintf(buffer, "&#%d;", (int)p[collpos]);
3630 for (cp = buffer; *cp; ++cp) {
3631 x = charmapencode_output(*cp, mapping, res, respos);
3632 if (x==enc_EXCEPTION)
3633 return -1;
3634 else if (x==enc_FAILED) {
3635 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3636 return -1;
3637 }
3638 }
3639 }
3640 *inpos = collendpos;
3641 break;
3642 default:
3643 repunicode = unicode_encode_call_errorhandler(errors, errorHandler,
3644 encoding, reason, p, size, exceptionObject,
3645 collstartpos, collendpos, &newpos);
3646 if (repunicode == NULL)
3647 return -1;
3648 /* generate replacement */
3649 repsize = PyUnicode_GET_SIZE(repunicode);
3650 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {
3651 x = charmapencode_output(*uni2, mapping, res, respos);
3652 if (x==enc_EXCEPTION) {
3653 return -1;
3654 }
3655 else if (x==enc_FAILED) {
3656 Py_DECREF(repunicode);
3657 raise_encode_exception(exceptionObject, encoding, p, size, collstartpos, collendpos, reason);
3658 return -1;
3659 }
3660 }
3661 *inpos = newpos;
3662 Py_DECREF(repunicode);
3663 }
3664 return 0;
3665}
3666
3667PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p,
3668 Py_ssize_t size,
3669 PyObject *mapping,
3670 const char *errors)
3671{
3672 /* output object */
3673 PyObject *res = NULL;
3674 /* current input position */
3675 Py_ssize_t inpos = 0;
3676 /* current output position */
3677 Py_ssize_t respos = 0;
3678 PyObject *errorHandler = NULL;
3679 PyObject *exc = NULL;
3680 /* the following variable is used for caching string comparisons
3681 * -1=not initialized, 0=unknown, 1=strict, 2=replace,
3682 * 3=ignore, 4=xmlcharrefreplace */
3683 int known_errorHandler = -1;
3684
3685 /* Default to Latin-1 */
3686 if (mapping == NULL)
3687 return PyUnicode_EncodeLatin1(p, size, errors);
3688
3689 /* allocate enough for a simple encoding without
3690 replacements, if we need more, we'll resize */
3691 res = PyString_FromStringAndSize(NULL, size);
3692 if (res == NULL)
3693 goto onError;
3694 if (size == 0)
3695 return res;
3696
3697 while (inpos<size) {
3698 /* try to encode it */
3699 charmapencode_result x = charmapencode_output(p[inpos], mapping, &res, &respos);
3700 if (x==enc_EXCEPTION) /* error */
3701 goto onError;
3702 if (x==enc_FAILED) { /* unencodable character */
3703 if (charmap_encoding_error(p, size, &inpos, mapping,
3704 &exc,
3705 &known_errorHandler, &errorHandler, errors,
3706 &res, &respos)) {
3707 goto onError;
3708 }
3709 }
3710 else
3711 /* done with this character => adjust input position */
3712 ++inpos;
3713 }
3714
3715 /* Resize if we allocated to much */
3716 if (respos<PyString_GET_SIZE(res)) {
3717 if (_PyString_Resize(&res, respos))
3718 goto onError;
3719 }
3720 Py_XDECREF(exc);
3721 Py_XDECREF(errorHandler);
3722 return res;
3723
3724 onError:
3725 Py_XDECREF(res);
3726 Py_XDECREF(exc);
3727 Py_XDECREF(errorHandler);
3728 return NULL;
3729}
3730
3731PyObject *PyUnicode_AsCharmapString(PyObject *unicode,
3732 PyObject *mapping)
3733{
3734 if (!PyUnicode_Check(unicode) || mapping == NULL) {
3735 PyErr_BadArgument();
3736 return NULL;
3737 }
3738 return PyUnicode_EncodeCharmap(PyUnicode_AS_UNICODE(unicode),
3739 PyUnicode_GET_SIZE(unicode),
3740 mapping,
3741 NULL);
3742}
3743
3744/* create or adjust a UnicodeTranslateError */
3745static void make_translate_exception(PyObject **exceptionObject,
3746 const Py_UNICODE *unicode, Py_ssize_t size,
3747 Py_ssize_t startpos, Py_ssize_t endpos,
3748 const char *reason)
3749{
3750 if (*exceptionObject == NULL) {
3751 *exceptionObject = PyUnicodeTranslateError_Create(
3752 unicode, size, startpos, endpos, reason);
3753 }
3754 else {
3755 if (PyUnicodeTranslateError_SetStart(*exceptionObject, startpos))
3756 goto onError;
3757 if (PyUnicodeTranslateError_SetEnd(*exceptionObject, endpos))
3758 goto onError;
3759 if (PyUnicodeTranslateError_SetReason(*exceptionObject, reason))
3760 goto onError;
3761 return;
3762 onError:
3763 Py_DECREF(*exceptionObject);
3764 *exceptionObject = NULL;
3765 }
3766}
3767
3768/* raises a UnicodeTranslateError */
3769static void raise_translate_exception(PyObject **exceptionObject,
3770 const Py_UNICODE *unicode, Py_ssize_t size,
3771 Py_ssize_t startpos, Py_ssize_t endpos,
3772 const char *reason)
3773{
3774 make_translate_exception(exceptionObject,
3775 unicode, size, startpos, endpos, reason);
3776 if (*exceptionObject != NULL)
3777 PyCodec_StrictErrors(*exceptionObject);
3778}
3779
3780/* error handling callback helper:
3781 build arguments, call the callback and check the arguments,
3782 put the result into newpos and return the replacement string, which
3783 has to be freed by the caller */
3784static PyObject *unicode_translate_call_errorhandler(const char *errors,
3785 PyObject **errorHandler,
3786 const char *reason,
3787 const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
3788 Py_ssize_t startpos, Py_ssize_t endpos,
3789 Py_ssize_t *newpos)
3790{
3791 static char *argparse = "O!n;translating error handler must return (unicode, int) tuple";
3792
3793 Py_ssize_t i_newpos;
3794 PyObject *restuple;
3795 PyObject *resunicode;
3796
3797 if (*errorHandler == NULL) {
3798 *errorHandler = PyCodec_LookupError(errors);
3799 if (*errorHandler == NULL)
3800 return NULL;
3801 }
3802
3803 make_translate_exception(exceptionObject,
3804 unicode, size, startpos, endpos, reason);
3805 if (*exceptionObject == NULL)
3806 return NULL;
3807
3808 restuple = PyObject_CallFunctionObjArgs(
3809 *errorHandler, *exceptionObject, NULL);
3810 if (restuple == NULL)
3811 return NULL;
3812 if (!PyTuple_Check(restuple)) {
3813 PyErr_Format(PyExc_TypeError, &argparse[4]);
3814 Py_DECREF(restuple);
3815 return NULL;
3816 }
3817 if (!PyArg_ParseTuple(restuple, argparse, &PyUnicode_Type,
3818 &resunicode, &i_newpos)) {
3819 Py_DECREF(restuple);
3820 return NULL;
3821 }
3822 if (i_newpos<0)
3823 *newpos = size+i_newpos;
3824 else
3825 *newpos = i_newpos;
3826 if (*newpos<0 || *newpos>size) {
3827 PyErr_Format(PyExc_IndexError, "position %zd from error handler out of bounds", *newpos);
3828 Py_DECREF(restuple);
3829 return NULL;
3830 }
3831 Py_INCREF(resunicode);
3832 Py_DECREF(restuple);
3833 return resunicode;
3834}
3835
3836/* Lookup the character ch in the mapping and put the result in result,
3837 which must be decrefed by the caller.
3838 Return 0 on success, -1 on error */
3839static
3840int charmaptranslate_lookup(Py_UNICODE c, PyObject *mapping, PyObject **result)
3841{
3842 PyObject *w = PyInt_FromLong((long)c);
3843 PyObject *x;
3844
3845 if (w == NULL)
3846 return -1;
3847 x = PyObject_GetItem(mapping, w);
3848 Py_DECREF(w);
3849 if (x == NULL) {
3850 if (PyErr_ExceptionMatches(PyExc_LookupError)) {
3851 /* No mapping found means: use 1:1 mapping. */
3852 PyErr_Clear();
3853 *result = NULL;
3854 return 0;
3855 } else
3856 return -1;
3857 }
3858 else if (x == Py_None) {
3859 *result = x;
3860 return 0;
3861 }
3862 else if (PyInt_Check(x)) {
3863 long value = PyInt_AS_LONG(x);
3864 long max = PyUnicode_GetMax();
3865 if (value < 0 || value > max) {
3866 PyErr_Format(PyExc_TypeError,
3867 "character mapping must be in range(0x%lx)", max+1);
3868 Py_DECREF(x);
3869 return -1;
3870 }
3871 *result = x;
3872 return 0;
3873 }
3874 else if (PyUnicode_Check(x)) {
3875 *result = x;
3876 return 0;
3877 }
3878 else {
3879 /* wrong return value */
3880 PyErr_SetString(PyExc_TypeError,
3881 "character mapping must return integer, None or unicode");
3882 Py_DECREF(x);
3883 return -1;
3884 }
3885}
3886/* ensure that *outobj is at least requiredsize characters long,
3887if not reallocate and adjust various state variables.
3888Return 0 on success, -1 on error */
3889static
3890int charmaptranslate_makespace(PyObject **outobj, Py_UNICODE **outp,
3891 Py_ssize_t requiredsize)
3892{
3893 Py_ssize_t oldsize = PyUnicode_GET_SIZE(*outobj);
3894 if (requiredsize > oldsize) {
3895 /* remember old output position */
3896 Py_ssize_t outpos = *outp-PyUnicode_AS_UNICODE(*outobj);
3897 /* exponentially overallocate to minimize reallocations */
3898 if (requiredsize < 2 * oldsize)
3899 requiredsize = 2 * oldsize;
3900 if (_PyUnicode_Resize(outobj, requiredsize) < 0)
3901 return -1;
3902 *outp = PyUnicode_AS_UNICODE(*outobj) + outpos;
3903 }
3904 return 0;
3905}
3906/* lookup the character, put the result in the output string and adjust
3907 various state variables. Return a new reference to the object that
3908 was put in the output buffer in *result, or Py_None, if the mapping was
3909 undefined (in which case no character was written).
3910 The called must decref result.
3911 Return 0 on success, -1 on error. */
3912static
3913int charmaptranslate_output(const Py_UNICODE *startinp, const Py_UNICODE *curinp,
3914 Py_ssize_t insize, PyObject *mapping, PyObject **outobj, Py_UNICODE **outp,
3915 PyObject **res)
3916{
3917 if (charmaptranslate_lookup(*curinp, mapping, res))
3918 return -1;
3919 if (*res==NULL) {
3920 /* not found => default to 1:1 mapping */
3921 *(*outp)++ = *curinp;
3922 }
3923 else if (*res==Py_None)
3924 ;
3925 else if (PyInt_Check(*res)) {
3926 /* no overflow check, because we know that the space is enough */
3927 *(*outp)++ = (Py_UNICODE)PyInt_AS_LONG(*res);
3928 }
3929 else if (PyUnicode_Check(*res)) {
3930 Py_ssize_t repsize = PyUnicode_GET_SIZE(*res);
3931 if (repsize==1) {
3932 /* no overflow check, because we know that the space is enough */
3933 *(*outp)++ = *PyUnicode_AS_UNICODE(*res);
3934 }
3935 else if (repsize!=0) {
3936 /* more than one character */
3937 Py_ssize_t requiredsize = (*outp-PyUnicode_AS_UNICODE(*outobj)) +
3938 (insize - (curinp-startinp)) +
3939 repsize - 1;
3940 if (charmaptranslate_makespace(outobj, outp, requiredsize))
3941 return -1;
3942 memcpy(*outp, PyUnicode_AS_UNICODE(*res), sizeof(Py_UNICODE)*repsize);
3943 *outp += repsize;
3944 }
3945 }
3946 else
3947 return -1;
3948 return 0;
3949}
3950
3951PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p,
3952 Py_ssize_t size,
3953 PyObject *mapping,
3954 const char *errors)
3955{
3956 /* output object */
3957 PyObject *res = NULL;
3958 /* pointers to the beginning and end+1 of input */
3959 const Py_UNICODE *startp = p;
3960 const Py_UNICODE *endp = p + size;
3961 /* pointer into the output */
3962 Py_UNICODE *str;
3963 /* current output position */
3964 Py_ssize_t respos = 0;
3965 char *reason = "character maps to <undefined>";
3966 PyObject *errorHandler = NULL;
3967 PyObject *exc = NULL;
3968 /* the following variable is used for caching string comparisons
3969 * -1=not initialized, 0=unknown, 1=strict, 2=replace,
3970 * 3=ignore, 4=xmlcharrefreplace */
3971 int known_errorHandler = -1;
3972
3973 if (mapping == NULL) {
3974 PyErr_BadArgument();
3975 return NULL;
3976 }
3977
3978 /* allocate enough for a simple 1:1 translation without
3979 replacements, if we need more, we'll resize */
3980 res = PyUnicode_FromUnicode(NULL, size);
3981 if (res == NULL)
3982 goto onError;
3983 if (size == 0)
3984 return res;
3985 str = PyUnicode_AS_UNICODE(res);
3986
3987 while (p<endp) {
3988 /* try to encode it */
3989 PyObject *x = NULL;
3990 if (charmaptranslate_output(startp, p, size, mapping, &res, &str, &x)) {
3991 Py_XDECREF(x);
3992 goto onError;
3993 }
3994 Py_XDECREF(x);
3995 if (x!=Py_None) /* it worked => adjust input pointer */
3996 ++p;
3997 else { /* untranslatable character */
3998 PyObject *repunicode = NULL; /* initialize to prevent gcc warning */
3999 Py_ssize_t repsize;
4000 Py_ssize_t newpos;
4001 Py_UNICODE *uni2;
4002 /* startpos for collecting untranslatable chars */
4003 const Py_UNICODE *collstart = p;
4004 const Py_UNICODE *collend = p+1;
4005 const Py_UNICODE *coll;
4006
4007 /* find all untranslatable characters */
4008 while (collend < endp) {
4009 if (charmaptranslate_lookup(*collend, mapping, &x))
4010 goto onError;
4011 Py_XDECREF(x);
4012 if (x!=Py_None)
4013 break;
4014 ++collend;
4015 }
4016 /* cache callback name lookup
4017 * (if not done yet, i.e. it's the first error) */
4018 if (known_errorHandler==-1) {
4019 if ((errors==NULL) || (!strcmp(errors, "strict")))
4020 known_errorHandler = 1;
4021 else if (!strcmp(errors, "replace"))
4022 known_errorHandler = 2;
4023 else if (!strcmp(errors, "ignore"))
4024 known_errorHandler = 3;
4025 else if (!strcmp(errors, "xmlcharrefreplace"))
4026 known_errorHandler = 4;
4027 else
4028 known_errorHandler = 0;
4029 }
4030 switch (known_errorHandler) {
4031 case 1: /* strict */
4032 raise_translate_exception(&exc, startp, size, collstart-startp, collend-startp, reason);
4033 goto onError;
4034 case 2: /* replace */
4035 /* No need to check for space, this is a 1:1 replacement */
4036 for (coll = collstart; coll<collend; ++coll)
4037 *str++ = '?';
4038 /* fall through */
4039 case 3: /* ignore */
4040 p = collend;
4041 break;
4042 case 4: /* xmlcharrefreplace */
4043 /* generate replacement (temporarily (mis)uses p) */
4044 for (p = collstart; p < collend; ++p) {
4045 char buffer[2+29+1+1];
4046 char *cp;
4047 sprintf(buffer, "&#%d;", (int)*p);
4048 if (charmaptranslate_makespace(&res, &str,
4049 (str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend)))
4050 goto onError;
4051 for (cp = buffer; *cp; ++cp)
4052 *str++ = *cp;
4053 }
4054 p = collend;
4055 break;
4056 default:
4057 repunicode = unicode_translate_call_errorhandler(errors, &errorHandler,
4058 reason, startp, size, &exc,
4059 collstart-startp, collend-startp, &newpos);
4060 if (repunicode == NULL)
4061 goto onError;
4062 /* generate replacement */
4063 repsize = PyUnicode_GET_SIZE(repunicode);
4064 if (charmaptranslate_makespace(&res, &str,
4065 (str-PyUnicode_AS_UNICODE(res))+repsize+(endp-collend))) {
4066 Py_DECREF(repunicode);
4067 goto onError;
4068 }
4069 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2)
4070 *str++ = *uni2;
4071 p = startp + newpos;
4072 Py_DECREF(repunicode);
4073 }
4074 }
4075 }
4076 /* Resize if we allocated to much */
4077 respos = str-PyUnicode_AS_UNICODE(res);
4078 if (respos<PyUnicode_GET_SIZE(res)) {
4079 if (_PyUnicode_Resize(&res, respos) < 0)
4080 goto onError;
4081 }
4082 Py_XDECREF(exc);
4083 Py_XDECREF(errorHandler);
4084 return res;
4085
4086 onError:
4087 Py_XDECREF(res);
4088 Py_XDECREF(exc);
4089 Py_XDECREF(errorHandler);
4090 return NULL;
4091}
4092
4093PyObject *PyUnicode_Translate(PyObject *str,
4094 PyObject *mapping,
4095 const char *errors)
4096{
4097 PyObject *result;
4098
4099 str = PyUnicode_FromObject(str);
4100 if (str == NULL)
4101 goto onError;
4102 result = PyUnicode_TranslateCharmap(PyUnicode_AS_UNICODE(str),
4103 PyUnicode_GET_SIZE(str),
4104 mapping,
4105 errors);
4106 Py_DECREF(str);
4107 return result;
4108
4109 onError:
4110 Py_XDECREF(str);
4111 return NULL;
4112}
4113
4114/* --- Decimal Encoder ---------------------------------------------------- */
4115
4116int PyUnicode_EncodeDecimal(Py_UNICODE *s,
4117 Py_ssize_t length,
4118 char *output,
4119 const char *errors)
4120{
4121 Py_UNICODE *p, *end;
4122 PyObject *errorHandler = NULL;
4123 PyObject *exc = NULL;
4124 const char *encoding = "decimal";
4125 const char *reason = "invalid decimal Unicode string";
4126 /* the following variable is used for caching string comparisons
4127 * -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
4128 int known_errorHandler = -1;
4129
4130 if (output == NULL) {
4131 PyErr_BadArgument();
4132 return -1;
4133 }
4134
4135 p = s;
4136 end = s + length;
4137 while (p < end) {
4138 register Py_UNICODE ch = *p;
4139 int decimal;
4140 PyObject *repunicode;
4141 Py_ssize_t repsize;
4142 Py_ssize_t newpos;
4143 Py_UNICODE *uni2;
4144 Py_UNICODE *collstart;
4145 Py_UNICODE *collend;
4146
4147 if (Py_UNICODE_ISSPACE(ch)) {
4148 *output++ = ' ';
4149 ++p;
4150 continue;
4151 }
4152 decimal = Py_UNICODE_TODECIMAL(ch);
4153 if (decimal >= 0) {
4154 *output++ = '0' + decimal;
4155 ++p;
4156 continue;
4157 }
4158 if (0 < ch && ch < 256) {
4159 *output++ = (char)ch;
4160 ++p;
4161 continue;
4162 }
4163 /* All other characters are considered unencodable */
4164 collstart = p;
4165 collend = p+1;
4166 while (collend < end) {
4167 if ((0 < *collend && *collend < 256) ||
4168 !Py_UNICODE_ISSPACE(*collend) ||
4169 Py_UNICODE_TODECIMAL(*collend))
4170 break;
4171 }
4172 /* cache callback name lookup
4173 * (if not done yet, i.e. it's the first error) */
4174 if (known_errorHandler==-1) {
4175 if ((errors==NULL) || (!strcmp(errors, "strict")))
4176 known_errorHandler = 1;
4177 else if (!strcmp(errors, "replace"))
4178 known_errorHandler = 2;
4179 else if (!strcmp(errors, "ignore"))
4180 known_errorHandler = 3;
4181 else if (!strcmp(errors, "xmlcharrefreplace"))
4182 known_errorHandler = 4;
4183 else
4184 known_errorHandler = 0;
4185 }
4186 switch (known_errorHandler) {
4187 case 1: /* strict */
4188 raise_encode_exception(&exc, encoding, s, length, collstart-s, collend-s, reason);
4189 goto onError;
4190 case 2: /* replace */
4191 for (p = collstart; p < collend; ++p)
4192 *output++ = '?';
4193 /* fall through */
4194 case 3: /* ignore */
4195 p = collend;
4196 break;
4197 case 4: /* xmlcharrefreplace */
4198 /* generate replacement (temporarily (mis)uses p) */
4199 for (p = collstart; p < collend; ++p)
4200 output += sprintf(output, "&#%d;", (int)*p);
4201 p = collend;
4202 break;
4203 default:
4204 repunicode = unicode_encode_call_errorhandler(errors, &errorHandler,
4205 encoding, reason, s, length, &exc,
4206 collstart-s, collend-s, &newpos);
4207 if (repunicode == NULL)
4208 goto onError;
4209 /* generate replacement */
4210 repsize = PyUnicode_GET_SIZE(repunicode);
4211 for (uni2 = PyUnicode_AS_UNICODE(repunicode); repsize-->0; ++uni2) {
4212 Py_UNICODE ch = *uni2;
4213 if (Py_UNICODE_ISSPACE(ch))
4214 *output++ = ' ';
4215 else {
4216 decimal = Py_UNICODE_TODECIMAL(ch);
4217 if (decimal >= 0)
4218 *output++ = '0' + decimal;
4219 else if (0 < ch && ch < 256)
4220 *output++ = (char)ch;
4221 else {
4222 Py_DECREF(repunicode);
4223 raise_encode_exception(&exc, encoding,
4224 s, length, collstart-s, collend-s, reason);
4225 goto onError;
4226 }
4227 }
4228 }
4229 p = s + newpos;
4230 Py_DECREF(repunicode);
4231 }
4232 }
4233 /* 0-terminate the output string */
4234 *output++ = '\0';
4235 Py_XDECREF(exc);
4236 Py_XDECREF(errorHandler);
4237 return 0;
4238
4239 onError:
4240 Py_XDECREF(exc);
4241 Py_XDECREF(errorHandler);
4242 return -1;
4243}
4244
4245/* --- Helpers ------------------------------------------------------------ */
4246
4247#define STRINGLIB_CHAR Py_UNICODE
4248
4249#define STRINGLIB_LEN PyUnicode_GET_SIZE
4250#define STRINGLIB_NEW PyUnicode_FromUnicode
4251#define STRINGLIB_STR PyUnicode_AS_UNICODE
4252
4253Py_LOCAL_INLINE(int)
4254STRINGLIB_CMP(const Py_UNICODE* str, const Py_UNICODE* other, Py_ssize_t len)
4255{
4256 if (str[0] != other[0])
4257 return 1;
4258 return memcmp((void*) str, (void*) other, len * sizeof(Py_UNICODE));
4259}
4260
4261#define STRINGLIB_EMPTY unicode_empty
4262
4263#include "stringlib/fastsearch.h"
4264
4265#include "stringlib/count.h"
4266#include "stringlib/find.h"
4267#include "stringlib/partition.h"
4268
4269/* helper macro to fixup start/end slice values */
4270#define FIX_START_END(obj) \
4271 if (start < 0) \
4272 start += (obj)->length; \
4273 if (start < 0) \
4274 start = 0; \
4275 if (end > (obj)->length) \
4276 end = (obj)->length; \
4277 if (end < 0) \
4278 end += (obj)->length; \
4279 if (end < 0) \
4280 end = 0;
4281
4282Py_ssize_t PyUnicode_Count(PyObject *str,
4283 PyObject *substr,
4284 Py_ssize_t start,
4285 Py_ssize_t end)
4286{
4287 Py_ssize_t result;
4288 PyUnicodeObject* str_obj;
4289 PyUnicodeObject* sub_obj;
4290
4291 str_obj = (PyUnicodeObject*) PyUnicode_FromObject(str);
4292 if (!str_obj)
4293 return -1;
4294 sub_obj = (PyUnicodeObject*) PyUnicode_FromObject(substr);
4295 if (!sub_obj) {
4296 Py_DECREF(str_obj);
4297 return -1;
4298 }
4299
4300 FIX_START_END(str_obj);
4301
4302 result = stringlib_count(
4303 str_obj->str + start, end - start, sub_obj->str, sub_obj->length
4304 );
4305
4306 Py_DECREF(sub_obj);
4307 Py_DECREF(str_obj);
4308
4309 return result;
4310}
4311
4312Py_ssize_t PyUnicode_Find(PyObject *str,
4313 PyObject *sub,
4314 Py_ssize_t start,
4315 Py_ssize_t end,
4316 int direction)
4317{
4318 Py_ssize_t result;
4319
4320 str = PyUnicode_FromObject(str);
4321 if (!str)
4322 return -2;
4323 sub = PyUnicode_FromObject(sub);
4324 if (!sub) {
4325 Py_DECREF(str);
4326 return -2;
4327 }
4328
4329 if (direction > 0)
4330 result = stringlib_find_slice(
4331 PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str),
4332 PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub),
4333 start, end
4334 );
4335 else
4336 result = stringlib_rfind_slice(
4337 PyUnicode_AS_UNICODE(str), PyUnicode_GET_SIZE(str),
4338 PyUnicode_AS_UNICODE(sub), PyUnicode_GET_SIZE(sub),
4339 start, end
4340 );
4341
4342 Py_DECREF(str);
4343 Py_DECREF(sub);
4344
4345 return result;
4346}
4347
4348static
4349int tailmatch(PyUnicodeObject *self,
4350 PyUnicodeObject *substring,
4351 Py_ssize_t start,
4352 Py_ssize_t end,
4353 int direction)
4354{
4355 if (substring->length == 0)
4356 return 1;
4357
4358 FIX_START_END(self);
4359
4360 end -= substring->length;
4361 if (end < start)
4362 return 0;
4363
4364 if (direction > 0) {
4365 if (Py_UNICODE_MATCH(self, end, substring))
4366 return 1;
4367 } else {
4368 if (Py_UNICODE_MATCH(self, start, substring))
4369 return 1;
4370 }
4371
4372 return 0;
4373}
4374
4375Py_ssize_t PyUnicode_Tailmatch(PyObject *str,
4376 PyObject *substr,
4377 Py_ssize_t start,
4378 Py_ssize_t end,
4379 int direction)
4380{
4381 Py_ssize_t result;
4382
4383 str = PyUnicode_FromObject(str);
4384 if (str == NULL)
4385 return -1;
4386 substr = PyUnicode_FromObject(substr);
4387 if (substr == NULL) {
4388 Py_DECREF(str);
4389 return -1;
4390 }
4391
4392 result = tailmatch((PyUnicodeObject *)str,
4393 (PyUnicodeObject *)substr,
4394 start, end, direction);
4395 Py_DECREF(str);
4396 Py_DECREF(substr);
4397 return result;
4398}
4399
4400/* Apply fixfct filter to the Unicode object self and return a
4401 reference to the modified object */
4402
4403static
4404PyObject *fixup(PyUnicodeObject *self,
4405 int (*fixfct)(PyUnicodeObject *s))
4406{
4407
4408 PyUnicodeObject *u;
4409
4410 u = (PyUnicodeObject*) PyUnicode_FromUnicode(NULL, self->length);
4411 if (u == NULL)
4412 return NULL;
4413
4414 Py_UNICODE_COPY(u->str, self->str, self->length);
4415
4416 if (!fixfct(u) && PyUnicode_CheckExact(self)) {
4417 /* fixfct should return TRUE if it modified the buffer. If
4418 FALSE, return a reference to the original buffer instead
4419 (to save space, not time) */
4420 Py_INCREF(self);
4421 Py_DECREF(u);
4422 return (PyObject*) self;
4423 }
4424 return (PyObject*) u;
4425}
4426
4427static
4428int fixupper(PyUnicodeObject *self)
4429{
4430 Py_ssize_t len = self->length;
4431 Py_UNICODE *s = self->str;
4432 int status = 0;
4433
4434 while (len-- > 0) {
4435 register Py_UNICODE ch;
4436
4437 ch = Py_UNICODE_TOUPPER(*s);
4438 if (ch != *s) {
4439 status = 1;
4440 *s = ch;
4441 }
4442 s++;
4443 }
4444
4445 return status;
4446}
4447
4448static
4449int fixlower(PyUnicodeObject *self)
4450{
4451 Py_ssize_t len = self->length;
4452 Py_UNICODE *s = self->str;
4453 int status = 0;
4454
4455 while (len-- > 0) {
4456 register Py_UNICODE ch;
4457
4458 ch = Py_UNICODE_TOLOWER(*s);
4459 if (ch != *s) {
4460 status = 1;
4461 *s = ch;
4462 }
4463 s++;
4464 }
4465
4466 return status;
4467}
4468
4469static
4470int fixswapcase(PyUnicodeObject *self)
4471{
4472 Py_ssize_t len = self->length;
4473 Py_UNICODE *s = self->str;
4474 int status = 0;
4475
4476 while (len-- > 0) {
4477 if (Py_UNICODE_ISUPPER(*s)) {
4478 *s = Py_UNICODE_TOLOWER(*s);
4479 status = 1;
4480 } else if (Py_UNICODE_ISLOWER(*s)) {
4481 *s = Py_UNICODE_TOUPPER(*s);
4482 status = 1;
4483 }
4484 s++;
4485 }
4486
4487 return status;
4488}
4489
4490static
4491int fixcapitalize(PyUnicodeObject *self)
4492{
4493 Py_ssize_t len = self->length;
4494 Py_UNICODE *s = self->str;
4495 int status = 0;
4496
4497 if (len == 0)
4498 return 0;
4499 if (Py_UNICODE_ISLOWER(*s)) {
4500 *s = Py_UNICODE_TOUPPER(*s);
4501 status = 1;
4502 }
4503 s++;
4504 while (--len > 0) {
4505 if (Py_UNICODE_ISUPPER(*s)) {
4506 *s = Py_UNICODE_TOLOWER(*s);
4507 status = 1;
4508 }
4509 s++;
4510 }
4511 return status;
4512}
4513
4514static
4515int fixtitle(PyUnicodeObject *self)
4516{
4517 register Py_UNICODE *p = PyUnicode_AS_UNICODE(self);
4518 register Py_UNICODE *e;
4519 int previous_is_cased;
4520
4521 /* Shortcut for single character strings */
4522 if (PyUnicode_GET_SIZE(self) == 1) {
4523 Py_UNICODE ch = Py_UNICODE_TOTITLE(*p);
4524 if (*p != ch) {
4525 *p = ch;
4526 return 1;
4527 }
4528 else
4529 return 0;
4530 }
4531
4532 e = p + PyUnicode_GET_SIZE(self);
4533 previous_is_cased = 0;
4534 for (; p < e; p++) {
4535 register const Py_UNICODE ch = *p;
4536
4537 if (previous_is_cased)
4538 *p = Py_UNICODE_TOLOWER(ch);
4539 else
4540 *p = Py_UNICODE_TOTITLE(ch);
4541
4542 if (Py_UNICODE_ISLOWER(ch) ||
4543 Py_UNICODE_ISUPPER(ch) ||
4544 Py_UNICODE_ISTITLE(ch))
4545 previous_is_cased = 1;
4546 else
4547 previous_is_cased = 0;
4548 }
4549 return 1;
4550}
4551
4552PyObject *
4553PyUnicode_Join(PyObject *separator, PyObject *seq)
4554{
4555 PyObject *internal_separator = NULL;
4556 const Py_UNICODE blank = ' ';
4557 const Py_UNICODE *sep = &blank;
4558 Py_ssize_t seplen = 1;
4559 PyUnicodeObject *res = NULL; /* the result */
4560 Py_ssize_t res_alloc = 100; /* # allocated bytes for string in res */
4561 Py_ssize_t res_used; /* # used bytes */
4562 Py_UNICODE *res_p; /* pointer to free byte in res's string area */
4563 PyObject *fseq; /* PySequence_Fast(seq) */
4564 Py_ssize_t seqlen; /* len(fseq) -- number of items in sequence */
4565 PyObject *item;
4566 Py_ssize_t i;
4567
4568 fseq = PySequence_Fast(seq, "");
4569 if (fseq == NULL) {
4570 return NULL;
4571 }
4572
4573 /* Grrrr. A codec may be invoked to convert str objects to
4574 * Unicode, and so it's possible to call back into Python code
4575 * during PyUnicode_FromObject(), and so it's possible for a sick
4576 * codec to change the size of fseq (if seq is a list). Therefore
4577 * we have to keep refetching the size -- can't assume seqlen
4578 * is invariant.
4579 */
4580 seqlen = PySequence_Fast_GET_SIZE(fseq);
4581 /* If empty sequence, return u"". */
4582 if (seqlen == 0) {
4583 res = _PyUnicode_New(0); /* empty sequence; return u"" */
4584 goto Done;
4585 }
4586 /* If singleton sequence with an exact Unicode, return that. */
4587 if (seqlen == 1) {
4588 item = PySequence_Fast_GET_ITEM(fseq, 0);
4589 if (PyUnicode_CheckExact(item)) {
4590 Py_INCREF(item);
4591 res = (PyUnicodeObject *)item;
4592 goto Done;
4593 }
4594 }
4595
4596 /* At least two items to join, or one that isn't exact Unicode. */
4597 if (seqlen > 1) {
4598 /* Set up sep and seplen -- they're needed. */
4599 if (separator == NULL) {
4600 sep = &blank;
4601 seplen = 1;
4602 }
4603 else {
4604 internal_separator = PyUnicode_FromObject(separator);
4605 if (internal_separator == NULL)
4606 goto onError;
4607 sep = PyUnicode_AS_UNICODE(internal_separator);
4608 seplen = PyUnicode_GET_SIZE(internal_separator);
4609 /* In case PyUnicode_FromObject() mutated seq. */
4610 seqlen = PySequence_Fast_GET_SIZE(fseq);
4611 }
4612 }
4613
4614 /* Get space. */
4615 res = _PyUnicode_New(res_alloc);
4616 if (res == NULL)
4617 goto onError;
4618 res_p = PyUnicode_AS_UNICODE(res);
4619 res_used = 0;
4620
4621 for (i = 0; i < seqlen; ++i) {
4622 Py_ssize_t itemlen;
4623 Py_ssize_t new_res_used;
4624
4625 item = PySequence_Fast_GET_ITEM(fseq, i);
4626 /* Convert item to Unicode. */
4627 if (! PyUnicode_Check(item) && ! PyString_Check(item)) {
4628 PyErr_Format(PyExc_TypeError,
4629 "sequence item %zd: expected string or Unicode,"
4630 " %.80s found",
4631 i, item->ob_type->tp_name);
4632 goto onError;
4633 }
4634 item = PyUnicode_FromObject(item);
4635 if (item == NULL)
4636 goto onError;
4637 /* We own a reference to item from here on. */
4638
4639 /* In case PyUnicode_FromObject() mutated seq. */
4640 seqlen = PySequence_Fast_GET_SIZE(fseq);
4641
4642 /* Make sure we have enough space for the separator and the item. */
4643 itemlen = PyUnicode_GET_SIZE(item);
4644 new_res_used = res_used + itemlen;
4645 if (new_res_used < 0)
4646 goto Overflow;
4647 if (i < seqlen - 1) {
4648 new_res_used += seplen;
4649 if (new_res_used < 0)
4650 goto Overflow;
4651 }
4652 if (new_res_used > res_alloc) {
4653 /* double allocated size until it's big enough */
4654 do {
4655 res_alloc += res_alloc;
4656 if (res_alloc <= 0)
4657 goto Overflow;
4658 } while (new_res_used > res_alloc);
4659 if (_PyUnicode_Resize(&res, res_alloc) < 0) {
4660 Py_DECREF(item);
4661 goto onError;
4662 }
4663 res_p = PyUnicode_AS_UNICODE(res) + res_used;
4664 }
4665
4666 /* Copy item, and maybe the separator. */
4667 Py_UNICODE_COPY(res_p, PyUnicode_AS_UNICODE(item), itemlen);
4668 res_p += itemlen;
4669 if (i < seqlen - 1) {
4670 Py_UNICODE_COPY(res_p, sep, seplen);
4671 res_p += seplen;
4672 }
4673 Py_DECREF(item);
4674 res_used = new_res_used;
4675 }
4676
4677 /* Shrink res to match the used area; this probably can't fail,
4678 * but it's cheap to check.
4679 */
4680 if (_PyUnicode_Resize(&res, res_used) < 0)
4681 goto onError;
4682
4683 Done:
4684 Py_XDECREF(internal_separator);
4685 Py_DECREF(fseq);
4686 return (PyObject *)res;
4687
4688 Overflow:
4689 PyErr_SetString(PyExc_OverflowError,
4690 "join() result is too long for a Python string");
4691 Py_DECREF(item);
4692 /* fall through */
4693
4694 onError:
4695 Py_XDECREF(internal_separator);
4696 Py_DECREF(fseq);
4697 Py_XDECREF(res);
4698 return NULL;
4699}
4700
4701static
4702PyUnicodeObject *pad(PyUnicodeObject *self,
4703 Py_ssize_t left,
4704 Py_ssize_t right,
4705 Py_UNICODE fill)
4706{
4707 PyUnicodeObject *u;
4708
4709 if (left < 0)
4710 left = 0;
4711 if (right < 0)
4712 right = 0;
4713
4714 if (left == 0 && right == 0 && PyUnicode_CheckExact(self)) {
4715 Py_INCREF(self);
4716 return self;
4717 }
4718
4719 u = _PyUnicode_New(left + self->length + right);
4720 if (u) {
4721 if (left)
4722 Py_UNICODE_FILL(u->str, fill, left);
4723 Py_UNICODE_COPY(u->str + left, self->str, self->length);
4724 if (right)
4725 Py_UNICODE_FILL(u->str + left + self->length, fill, right);
4726 }
4727
4728 return u;
4729}
4730
4731#define SPLIT_APPEND(data, left, right) \
4732 str = PyUnicode_FromUnicode((data) + (left), (right) - (left)); \
4733 if (!str) \
4734 goto onError; \
4735 if (PyList_Append(list, str)) { \
4736 Py_DECREF(str); \
4737 goto onError; \
4738 } \
4739 else \
4740 Py_DECREF(str);
4741
4742static
4743PyObject *split_whitespace(PyUnicodeObject *self,
4744 PyObject *list,
4745 Py_ssize_t maxcount)
4746{
4747 register Py_ssize_t i;
4748 register Py_ssize_t j;
4749 Py_ssize_t len = self->length;
4750 PyObject *str;
4751
4752 for (i = j = 0; i < len; ) {
4753 /* find a token */
4754 while (i < len && Py_UNICODE_ISSPACE(self->str[i]))
4755 i++;
4756 j = i;
4757 while (i < len && !Py_UNICODE_ISSPACE(self->str[i]))
4758 i++;
4759 if (j < i) {
4760 if (maxcount-- <= 0)
4761 break;
4762 SPLIT_APPEND(self->str, j, i);
4763 while (i < len && Py_UNICODE_ISSPACE(self->str[i]))
4764 i++;
4765 j = i;
4766 }
4767 }
4768 if (j < len) {
4769 SPLIT_APPEND(self->str, j, len);
4770 }
4771 return list;
4772
4773 onError:
4774 Py_DECREF(list);
4775 return NULL;
4776}
4777
4778PyObject *PyUnicode_Splitlines(PyObject *string,
4779 int keepends)
4780{
4781 register Py_ssize_t i;
4782 register Py_ssize_t j;
4783 Py_ssize_t len;
4784 PyObject *list;
4785 PyObject *str;
4786 Py_UNICODE *data;
4787
4788 string = PyUnicode_FromObject(string);
4789 if (string == NULL)
4790 return NULL;
4791 data = PyUnicode_AS_UNICODE(string);
4792 len = PyUnicode_GET_SIZE(string);
4793
4794 list = PyList_New(0);
4795 if (!list)
4796 goto onError;
4797
4798 for (i = j = 0; i < len; ) {
4799 Py_ssize_t eol;
4800
4801 /* Find a line and append it */
4802 while (i < len && !BLOOM_LINEBREAK(data[i]))
4803 i++;
4804
4805 /* Skip the line break reading CRLF as one line break */
4806 eol = i;
4807 if (i < len) {
4808 if (data[i] == '\r' && i + 1 < len &&
4809 data[i+1] == '\n')
4810 i += 2;
4811 else
4812 i++;
4813 if (keepends)
4814 eol = i;
4815 }
4816 SPLIT_APPEND(data, j, eol);
4817 j = i;
4818 }
4819 if (j < len) {
4820 SPLIT_APPEND(data, j, len);
4821 }
4822
4823 Py_DECREF(string);
4824 return list;
4825
4826 onError:
4827 Py_XDECREF(list);
4828 Py_DECREF(string);
4829 return NULL;
4830}
4831
4832static
4833PyObject *split_char(PyUnicodeObject *self,
4834 PyObject *list,
4835 Py_UNICODE ch,
4836 Py_ssize_t maxcount)
4837{
4838 register Py_ssize_t i;
4839 register Py_ssize_t j;
4840 Py_ssize_t len = self->length;
4841 PyObject *str;
4842
4843 for (i = j = 0; i < len; ) {
4844 if (self->str[i] == ch) {
4845 if (maxcount-- <= 0)
4846 break;
4847 SPLIT_APPEND(self->str, j, i);
4848 i = j = i + 1;
4849 } else
4850 i++;
4851 }
4852 if (j <= len) {
4853 SPLIT_APPEND(self->str, j, len);
4854 }
4855 return list;
4856
4857 onError:
4858 Py_DECREF(list);
4859 return NULL;
4860}
4861
4862static
4863PyObject *split_substring(PyUnicodeObject *self,
4864 PyObject *list,
4865 PyUnicodeObject *substring,
4866 Py_ssize_t maxcount)
4867{
4868 register Py_ssize_t i;
4869 register Py_ssize_t j;
4870 Py_ssize_t len = self->length;
4871 Py_ssize_t sublen = substring->length;
4872 PyObject *str;
4873
4874 for (i = j = 0; i <= len - sublen; ) {
4875 if (Py_UNICODE_MATCH(self, i, substring)) {
4876 if (maxcount-- <= 0)
4877 break;
4878 SPLIT_APPEND(self->str, j, i);
4879 i = j = i + sublen;
4880 } else
4881 i++;
4882 }
4883 if (j <= len) {
4884 SPLIT_APPEND(self->str, j, len);
4885 }
4886 return list;
4887
4888 onError:
4889 Py_DECREF(list);
4890 return NULL;
4891}
4892
4893static
4894PyObject *rsplit_whitespace(PyUnicodeObject *self,
4895 PyObject *list,
4896 Py_ssize_t maxcount)
4897{
4898 register Py_ssize_t i;
4899 register Py_ssize_t j;
4900 Py_ssize_t len = self->length;
4901 PyObject *str;
4902
4903 for (i = j = len - 1; i >= 0; ) {
4904 /* find a token */
4905 while (i >= 0 && Py_UNICODE_ISSPACE(self->str[i]))
4906 i--;
4907 j = i;
4908 while (i >= 0 && !Py_UNICODE_ISSPACE(self->str[i]))
4909 i--;
4910 if (j > i) {
4911 if (maxcount-- <= 0)
4912 break;
4913 SPLIT_APPEND(self->str, i + 1, j + 1);
4914 while (i >= 0 && Py_UNICODE_ISSPACE(self->str[i]))
4915 i--;
4916 j = i;
4917 }
4918 }
4919 if (j >= 0) {
4920 SPLIT_APPEND(self->str, 0, j + 1);
4921 }
4922 if (PyList_Reverse(list) < 0)
4923 goto onError;
4924 return list;
4925
4926 onError:
4927 Py_DECREF(list);
4928 return NULL;
4929}
4930
4931static
4932PyObject *rsplit_char(PyUnicodeObject *self,
4933 PyObject *list,
4934 Py_UNICODE ch,
4935 Py_ssize_t maxcount)
4936{
4937 register Py_ssize_t i;
4938 register Py_ssize_t j;
4939 Py_ssize_t len = self->length;
4940 PyObject *str;
4941
4942 for (i = j = len - 1; i >= 0; ) {
4943 if (self->str[i] == ch) {
4944 if (maxcount-- <= 0)
4945 break;
4946 SPLIT_APPEND(self->str, i + 1, j + 1);
4947 j = i = i - 1;
4948 } else
4949 i--;
4950 }
4951 if (j >= -1) {
4952 SPLIT_APPEND(self->str, 0, j + 1);
4953 }
4954 if (PyList_Reverse(list) < 0)
4955 goto onError;
4956 return list;
4957
4958 onError:
4959 Py_DECREF(list);
4960 return NULL;
4961}
4962
4963static
4964PyObject *rsplit_substring(PyUnicodeObject *self,
4965 PyObject *list,
4966 PyUnicodeObject *substring,
4967 Py_ssize_t maxcount)
4968{
4969 register Py_ssize_t i;
4970 register Py_ssize_t j;
4971 Py_ssize_t len = self->length;
4972 Py_ssize_t sublen = substring->length;
4973 PyObject *str;
4974
4975 for (i = len - sublen, j = len; i >= 0; ) {
4976 if (Py_UNICODE_MATCH(self, i, substring)) {
4977 if (maxcount-- <= 0)
4978 break;
4979 SPLIT_APPEND(self->str, i + sublen, j);
4980 j = i;
4981 i -= sublen;
4982 } else
4983 i--;
4984 }
4985 if (j >= 0) {
4986 SPLIT_APPEND(self->str, 0, j);
4987 }
4988 if (PyList_Reverse(list) < 0)
4989 goto onError;
4990 return list;
4991
4992 onError:
4993 Py_DECREF(list);
4994 return NULL;
4995}
4996
4997#undef SPLIT_APPEND
4998
4999static

3068 more lines aren’t shown here.