""" codecs -- Python Codec Registry, API and helpers.  Written by Marc-Andre Lemburg (mal@lemburg.com). (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. """#" import __builtin__, sys ### Registry and builtin stateless codec functions try:    from _codecs import *except ImportError, why:    raise SystemError('Failed to load the builtin codecs: %s' % why) __all__ = ["register", "lookup", "open", "EncodedFile", "BOM", "BOM_BE",           "BOM_LE", "BOM32_BE", "BOM32_LE", "BOM64_BE", "BOM64_LE",           "BOM_UTF8", "BOM_UTF16", "BOM_UTF16_LE", "BOM_UTF16_BE",           "BOM_UTF32", "BOM_UTF32_LE", "BOM_UTF32_BE",           "strict_errors", "ignore_errors", "replace_errors",           "xmlcharrefreplace_errors",           "register_error", "lookup_error"] ### Constants ## Byte Order Mark (BOM = ZERO WIDTH NO-BREAK SPACE = U+FEFF)# and its possible byte string values# for UTF8/UTF16/UTF32 output and little/big endian machines# # UTF-8BOM_UTF8 = '\xef\xbb\xbf' # UTF-16, little endianBOM_LE = BOM_UTF16_LE = '\xff\xfe' # UTF-16, big endianBOM_BE = BOM_UTF16_BE = '\xfe\xff' # UTF-32, little endianBOM_UTF32_LE = '\xff\xfe\x00\x00' # UTF-32, big endianBOM_UTF32_BE = '\x00\x00\xfe\xff' if sys.byteorder == 'little':     # UTF-16, native endianness    BOM = BOM_UTF16 = BOM_UTF16_LE     # UTF-32, native endianness    BOM_UTF32 = BOM_UTF32_LE else:     # UTF-16, native endianness    BOM = BOM_UTF16 = BOM_UTF16_BE     # UTF-32, native endianness    BOM_UTF32 = BOM_UTF32_BE # Old broken names (don't use in new code)BOM32_LE = BOM_UTF16_LEBOM32_BE = BOM_UTF16_BEBOM64_LE = BOM_UTF32_LEBOM64_BE = BOM_UTF32_BE  ### Codec base classes (defining the API) class CodecInfo(tuple):     def __new__(cls, encode, decode, streamreader=None, streamwriter=None,        incrementalencoder=None, incrementaldecoder=None, name=None):        self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter))        self.name = name        self.encode = encode        self.decode = decode        self.incrementalencoder = incrementalencoder        self.incrementaldecoder = incrementaldecoder        self.streamwriter = streamwriter        self.streamreader = streamreader        return self     def __repr__(self):        return "<%s.%s object for encoding %s at 0x%x>" % (self.__class__.__module__, self.__class__.__name__, self.name, id(self)) class Codec:     """ Defines the interface for stateless encoders/decoders.         The .encode()/.decode() methods may use different error        handling schemes by providing the errors argument. These        string values are predefined:          'strict' - raise a ValueError error (or a subclass)         'ignore' - ignore the character and continue with the next         'replace' - replace with a suitable replacement character;                    Python will use the official U+FFFD REPLACEMENT                    CHARACTER for the builtin Unicode codecs on                    decoding and '?' on encoding.         'xmlcharrefreplace' - Replace with the appropriate XML                               character reference (only for encoding).         'backslashreplace'  - Replace with backslashed escape sequences                               (only for encoding).         The set of allowed values can be extended via register_error.     """    def encode(self, input, errors='strict'):         """ Encodes the object input and returns a tuple (output            object, length consumed).             errors defines the error handling to apply. It defaults to            'strict' handling.             The method may not store state in the Codec instance. Use            StreamCodec for codecs which have to keep state in order to            make encoding/decoding efficient.             The encoder must be able to handle zero length input and            return an empty object of the output object type in this            situation.         """        raise NotImplementedError     def decode(self, input, errors='strict'):         """ Decodes the object input and returns a tuple (output            object, length consumed).             input must be an object which provides the bf_getreadbuf            buffer slot. Python strings, buffer objects and memory            mapped files are examples of objects providing this slot.             errors defines the error handling to apply. It defaults to            'strict' handling.             The method may not store state in the Codec instance. Use            StreamCodec for codecs which have to keep state in order to            make encoding/decoding efficient.             The decoder must be able to handle zero length input and            return an empty object of the output object type in this            situation.         """        raise NotImplementedError class IncrementalEncoder(object):    """    An IncrementalEncoder encodes an input in multiple steps. The input can be    passed piece by piece to the encode() method. The IncrementalEncoder remembers    the state of the Encoding process between calls to encode().    """    def __init__(self, errors='strict'):        """        Creates an IncrementalEncoder instance.         The IncrementalEncoder may use different error handling schemes by        providing the errors keyword argument. See the module docstring        for a list of possible values.        """        self.errors = errors        self.buffer = ""     def encode(self, input, final=False):        """        Encodes input and returns the resulting object.        """        raise NotImplementedError     def reset(self):        """        Resets the encoder to the initial state.        """ class BufferedIncrementalEncoder(IncrementalEncoder):    """    This subclass of IncrementalEncoder can be used as the baseclass for an    incremental encoder if the encoder must keep some of the output in a    buffer between calls to encode().    """    def __init__(self, errors='strict'):        IncrementalEncoder.__init__(self, errors)        self.buffer = "" # unencoded input that is kept between calls to encode()     def _buffer_encode(self, input, errors, final):        # Overwrite this method in subclasses: It must encode input        # and return an (output, length consumed) tuple        raise NotImplementedError     def encode(self, input, final=False):        # encode input (taking the buffer into account)        data = self.buffer + input        (result, consumed) = self._buffer_encode(data, self.errors, final)        # keep unencoded input until the next call        self.buffer = data[consumed:]        return result     def reset(self):        IncrementalEncoder.reset(self)        self.buffer = "" class IncrementalDecoder(object):    """    An IncrementalDecoder decodes an input in multiple steps. The input can be    passed piece by piece to the decode() method. The IncrementalDecoder    remembers the state of the decoding process between calls to decode().    """    def __init__(self, errors='strict'):        """        Creates a IncrementalDecoder instance.         The IncrementalDecoder may use different error handling schemes by        providing the errors keyword argument. See the module docstring        for a list of possible values.        """        self.errors = errors     def decode(self, input, final=False):        """        Decodes input and returns the resulting object.        """        raise NotImplementedError     def reset(self):        """        Resets the decoder to the initial state.        """ class BufferedIncrementalDecoder(IncrementalDecoder):    """    This subclass of IncrementalDecoder can be used as the baseclass for an    incremental decoder if the decoder must be able to handle incomplete byte    sequences.    """    def __init__(self, errors='strict'):        IncrementalDecoder.__init__(self, errors)        self.buffer = "" # undecoded input that is kept between calls to decode()     def _buffer_decode(self, input, errors, final):        # Overwrite this method in subclasses: It must decode input        # and return an (output, length consumed) tuple        raise NotImplementedError     def decode(self, input, final=False):        # decode input (taking the buffer into account)        data = self.buffer + input        (result, consumed) = self._buffer_decode(data, self.errors, final)        # keep undecoded input until the next call        self.buffer = data[consumed:]        return result     def reset(self):        IncrementalDecoder.reset(self)        self.buffer = "" ## The StreamWriter and StreamReader class provide generic working# interfaces which can be used to implement new encoding submodules# very easily. See encodings/utf_8.py for an example on how this is# done.# class StreamWriter(Codec):     def __init__(self, stream, errors='strict'):         """ Creates a StreamWriter instance.             stream must be a file-like object open for writing            (binary) data.             The StreamWriter may use different error handling            schemes by providing the errors keyword argument. These            parameters are predefined:              'strict' - raise a ValueError (or a subclass)             'ignore' - ignore the character and continue with the next             'replace'- replace with a suitable replacement character             'xmlcharrefreplace' - Replace with the appropriate XML                                   character reference.             'backslashreplace'  - Replace with backslashed escape                                   sequences (only for encoding).             The set of allowed parameter values can be extended via            register_error.        """        self.stream = stream        self.errors = errors     def write(self, object):         """ Writes the object's contents encoded to self.stream.        """        data, consumed = self.encode(object, self.errors)        self.stream.write(data)     def writelines(self, list):         """ Writes the concatenated list of strings to the stream            using .write().        """        self.write(''.join(list))     def reset(self):         """ Flushes and resets the codec buffers used for keeping state.             Calling this method should ensure that the data on the            output is put into a clean state, that allows appending            of new fresh data without having to rescan the whole            stream to recover state.         """        pass     def __getattr__(self, name,                    getattr=getattr):         """ Inherit all other methods from the underlying stream.        """        return getattr(self.stream, name)     def __enter__(self):        return self     def __exit__(self, type, value, tb):        self.stream.close() ### class StreamReader(Codec):     def __init__(self, stream, errors='strict'):         """ Creates a StreamReader instance.             stream must be a file-like object open for reading            (binary) data.             The StreamReader may use different error handling            schemes by providing the errors keyword argument. These            parameters are predefined:              'strict' - raise a ValueError (or a subclass)             'ignore' - ignore the character and continue with the next             'replace'- replace with a suitable replacement character;             The set of allowed parameter values can be extended via            register_error.        """        self.stream = stream        self.errors = errors        self.bytebuffer = ""        # For str->str decoding this will stay a str        # For str->unicode decoding the first read will promote it to unicode        self.charbuffer = ""        self.linebuffer = None     def decode(self, input, errors='strict'):        raise NotImplementedError     def read(self, size=-1, chars=-1, firstline=False):         """ Decodes data from the stream self.stream and returns the            resulting object.             chars indicates the number of characters to read from the            stream. read() will never return more than chars            characters, but it might return less, if there are not enough            characters available.             size indicates the approximate maximum number of bytes to            read from the stream for decoding purposes. The decoder            can modify this setting as appropriate. The default value            -1 indicates to read and decode as much as possible.  size            is intended to prevent having to decode huge files in one            step.             If firstline is true, and a UnicodeDecodeError happens            after the first line terminator in the input only the first line            will be returned, the rest of the input will be kept until the            next call to read().             The method should use a greedy read strategy meaning that            it should read as much data as is allowed within the            definition of the encoding and the given size, e.g.  if            optional encoding endings or state markers are available            on the stream, these should be read too.        """        # If we have lines cached, first merge them back into characters        if self.linebuffer:            self.charbuffer = "".join(self.linebuffer)            self.linebuffer = None         # read until we get the required number of characters (if available)        while True:            # can the request can be satisfied from the character buffer?            if chars < 0:                if size < 0:                    if self.charbuffer:                        break                elif len(self.charbuffer) >= size:                    break            else:                if len(self.charbuffer) >= chars:                    break            # we need more data            if size < 0:                newdata = self.stream.read()            else:                newdata = self.stream.read(size)            # decode bytes (those remaining from the last call included)            data = self.bytebuffer + newdata            try:                newchars, decodedbytes = self.decode(data, self.errors)            except UnicodeDecodeError, exc:                if firstline:                    newchars, decodedbytes = self.decode(data[:exc.start], self.errors)                    lines = newchars.splitlines(True)                    if len(lines)<=1:                        raise                else:                    raise            # keep undecoded bytes until the next call            self.bytebuffer = data[decodedbytes:]            # put new characters in the character buffer            self.charbuffer += newchars            # there was no data available            if not newdata:                break        if chars < 0:            # Return everything we've got            result = self.charbuffer            self.charbuffer = ""        else:            # Return the first chars characters            result = self.charbuffer[:chars]            self.charbuffer = self.charbuffer[chars:]        return result     def readline(self, size=None, keepends=True):         """ Read one line from the input stream and return the            decoded data.             size, if given, is passed as size argument to the            read() method.         """        # If we have lines cached from an earlier read, return        # them unconditionally        if self.linebuffer:            line = self.linebuffer[0]            del self.linebuffer[0]            if len(self.linebuffer) == 1:                # revert to charbuffer mode; we might need more data                # next time                self.charbuffer = self.linebuffer[0]                self.linebuffer = None            if not keepends:                line = line.splitlines(False)[0]            return line         readsize = size or 72        line = ""        # If size is given, we call read() only once        while True:            data = self.read(readsize, firstline=True)            if data:                # If we're at a "\r" read one extra character (which might                # be a "\n") to get a proper line ending. If the stream is                # temporarily exhausted we return the wrong line ending.                if data.endswith("\r"):                    data += self.read(size=1, chars=1)             line += data            lines = line.splitlines(True)            if lines:                if len(lines) > 1:                    # More than one line result; the first line is a full line                    # to return                    line = lines[0]                    del lines[0]                    if len(lines) > 1:                        # cache the remaining lines                        lines[-1] += self.charbuffer                        self.linebuffer = lines                        self.charbuffer = None                    else:                        # only one remaining line, put it back into charbuffer                        self.charbuffer = lines[0] + self.charbuffer

Lines 1–500 of 1034. Load more