"""HTTP/1.1 client library <intro stuff goes here><other stuff, too> HTTPConnection goes through a number of "states", which define when a clientmay legally make another request or fetch the response for a particularrequest. This diagram details these state transitions:     (null)      |      | HTTPConnection()      v    Idle      |      | putrequest()      v    Request-started      |      | ( putheader() )*  endheaders()      v    Request-sent      |      | response = getresponse()      v    Unread-response   [Response-headers-read]      |\____________________      |                     |      | response.read()     | putrequest()      v                     v    Idle                  Req-started-unread-response                     ______/|                   /        |   response.read() |        | ( putheader() )*  endheaders()                   v        v       Request-started    Req-sent-unread-response                            |                            | response.read()                            v                          Request-sent This diagram presents the following rules:  -- a second request may not be started until {response-headers-read}  -- a response [object] cannot be retrieved until {request-sent}  -- there is no differentiation between an unread response body and a     partially read response body Note: this enforcement is applied by the HTTPConnection class. The      HTTPResponse class does not enforce this state machine, which      implies sophisticated clients may accelerate the request/response      pipeline. Caution should be taken, though: accelerating the states      beyond the above pattern may imply knowledge of the server's      connection-close behavior for certain requests. For example, it      is impossible to tell whether the server will close the connection      UNTIL the response headers have been read; this means that further      requests cannot be placed into the pipeline until it is known that      the server will NOT be closing the connection. Logical State                  __state            __response-------------                  -------            ----------Idle                           _CS_IDLE           NoneRequest-started                _CS_REQ_STARTED    NoneRequest-sent                   _CS_REQ_SENT       NoneUnread-response                _CS_IDLE           <response_class>Req-started-unread-response    _CS_REQ_STARTED    <response_class>Req-sent-unread-response       _CS_REQ_SENT       <response_class>""" import errnoimport mimetoolsimport socketfrom urlparse import urlsplit try:    from cStringIO import StringIOexcept ImportError:    from StringIO import StringIO __all__ = ["HTTP", "HTTPResponse", "HTTPConnection", "HTTPSConnection",           "HTTPException", "NotConnected", "UnknownProtocol",           "UnknownTransferEncoding", "UnimplementedFileMode",           "IncompleteRead", "InvalidURL", "ImproperConnectionState",           "CannotSendRequest", "CannotSendHeader", "ResponseNotReady",           "BadStatusLine", "error", "responses"] HTTP_PORT = 80HTTPS_PORT = 443 _UNKNOWN = 'UNKNOWN' # connection states_CS_IDLE = 'Idle'_CS_REQ_STARTED = 'Request-started'_CS_REQ_SENT = 'Request-sent' # status codes# informationalCONTINUE = 100SWITCHING_PROTOCOLS = 101PROCESSING = 102 # successfulOK = 200CREATED = 201ACCEPTED = 202NON_AUTHORITATIVE_INFORMATION = 203NO_CONTENT = 204RESET_CONTENT = 205PARTIAL_CONTENT = 206MULTI_STATUS = 207IM_USED = 226 # redirectionMULTIPLE_CHOICES = 300MOVED_PERMANENTLY = 301FOUND = 302SEE_OTHER = 303NOT_MODIFIED = 304USE_PROXY = 305TEMPORARY_REDIRECT = 307 # client errorBAD_REQUEST = 400UNAUTHORIZED = 401PAYMENT_REQUIRED = 402FORBIDDEN = 403NOT_FOUND = 404METHOD_NOT_ALLOWED = 405NOT_ACCEPTABLE = 406PROXY_AUTHENTICATION_REQUIRED = 407REQUEST_TIMEOUT = 408CONFLICT = 409GONE = 410LENGTH_REQUIRED = 411PRECONDITION_FAILED = 412REQUEST_ENTITY_TOO_LARGE = 413REQUEST_URI_TOO_LONG = 414UNSUPPORTED_MEDIA_TYPE = 415REQUESTED_RANGE_NOT_SATISFIABLE = 416EXPECTATION_FAILED = 417UNPROCESSABLE_ENTITY = 422LOCKED = 423FAILED_DEPENDENCY = 424UPGRADE_REQUIRED = 426 # server errorINTERNAL_SERVER_ERROR = 500NOT_IMPLEMENTED = 501BAD_GATEWAY = 502SERVICE_UNAVAILABLE = 503GATEWAY_TIMEOUT = 504HTTP_VERSION_NOT_SUPPORTED = 505INSUFFICIENT_STORAGE = 507NOT_EXTENDED = 510 # Mapping status codes to official W3C namesresponses = {    100: 'Continue',    101: 'Switching Protocols',     200: 'OK',    201: 'Created',    202: 'Accepted',    203: 'Non-Authoritative Information',    204: 'No Content',    205: 'Reset Content',    206: 'Partial Content',     300: 'Multiple Choices',    301: 'Moved Permanently',    302: 'Found',    303: 'See Other',    304: 'Not Modified',    305: 'Use Proxy',    306: '(Unused)',    307: 'Temporary Redirect',     400: 'Bad Request',    401: 'Unauthorized',    402: 'Payment Required',    403: 'Forbidden',    404: 'Not Found',    405: 'Method Not Allowed',    406: 'Not Acceptable',    407: 'Proxy Authentication Required',    408: 'Request Timeout',    409: 'Conflict',    410: 'Gone',    411: 'Length Required',    412: 'Precondition Failed',    413: 'Request Entity Too Large',    414: 'Request-URI Too Long',    415: 'Unsupported Media Type',    416: 'Requested Range Not Satisfiable',    417: 'Expectation Failed',     500: 'Internal Server Error',    501: 'Not Implemented',    502: 'Bad Gateway',    503: 'Service Unavailable',    504: 'Gateway Timeout',    505: 'HTTP Version Not Supported',} # maximal amount of data to read at one time in _safe_readMAXAMOUNT = 1048576 class HTTPMessage(mimetools.Message):     def addheader(self, key, value):        """Add header for field key handling repeats."""        prev = self.dict.get(key)        if prev is None:            self.dict[key] = value        else:            combined = ", ".join((prev, value))            self.dict[key] = combined     def addcontinue(self, key, more):        """Add more field data from a continuation line."""        prev = self.dict[key]        self.dict[key] = prev + "\n " + more     def readheaders(self):        """Read header lines.         Read header lines up to the entirely blank line that terminates them.        The (normally blank) line that ends the headers is skipped, but not        included in the returned list.  If a non-header line ends the headers,        (which is an error), an attempt is made to backspace over it; it is        never included in the returned list.         The variable self.status is set to the empty string if all went well,        otherwise it is an error message.  The variable self.headers is a        completely uninterpreted list of lines contained in the header (so        printing them will reproduce the header exactly as it appears in the        file).         If multiple header fields with the same name occur, they are combined        according to the rules in RFC 2616 sec 4.2:         Appending each subsequent field-value to the first, each separated        by a comma. The order in which header fields with the same field-name        are received is significant to the interpretation of the combined        field value.        """        # XXX The implementation overrides the readheaders() method of        # rfc822.Message.  The base class design isn't amenable to        # customized behavior here so the method here is a copy of the        # base class code with a few small changes.         self.dict = {}        self.unixfrom = ''        self.headers = hlist = []        self.status = ''        headerseen = ""        firstline = 1        startofline = unread = tell = None        if hasattr(self.fp, 'unread'):            unread = self.fp.unread        elif self.seekable:            tell = self.fp.tell        while True:            if tell:                try:                    startofline = tell()                except IOError:                    startofline = tell = None                    self.seekable = 0            line = self.fp.readline()            if not line:                self.status = 'EOF in headers'                break            # Skip unix From name time lines            if firstline and line.startswith('From '):                self.unixfrom = self.unixfrom + line                continue            firstline = 0            if headerseen and line[0] in ' \t':                # XXX Not sure if continuation lines are handled properly                # for http and/or for repeating headers                # It's a continuation line.                hlist.append(line)                self.addcontinue(headerseen, line.strip())                continue            elif self.iscomment(line):                # It's a comment.  Ignore it.                continue            elif self.islast(line):                # Note! No pushback here!  The delimiter line gets eaten.                break            headerseen = self.isheader(line)            if headerseen:                # It's a legal header line, save it.                hlist.append(line)                self.addheader(headerseen, line[len(headerseen)+1:].strip())                continue            else:                # It's not a header line; throw it back and stop here.                if not self.dict:                    self.status = 'No headers'                else:                    self.status = 'Non-header line where header expected'                # Try to undo the read.                if unread:                    unread(line)                elif tell:                    self.fp.seek(startofline)                else:                    self.status = self.status + '; bad seek'                break class HTTPResponse:     # strict: If true, raise BadStatusLine if the status line can't be    # parsed as a valid HTTP/1.0 or 1.1 status line.  By default it is    # false because it prevents clients from talking to HTTP/0.9    # servers.  Note that a response with a sufficiently corrupted    # status line will look like an HTTP/0.9 response.     # See RFC 2616 sec 19.6 and RFC 1945 sec 6 for details.     def __init__(self, sock, debuglevel=0, strict=0, method=None):        self.fp = sock.makefile('rb', 0)        self.debuglevel = debuglevel        self.strict = strict        self._method = method         self.msg = None         # from the Status-Line of the response        self.version = _UNKNOWN # HTTP-Version        self.status = _UNKNOWN  # Status-Code        self.reason = _UNKNOWN  # Reason-Phrase         self.chunked = _UNKNOWN         # is "chunked" being used?        self.chunk_left = _UNKNOWN      # bytes left to read in current chunk        self.length = _UNKNOWN          # number of bytes left in response        self.will_close = _UNKNOWN      # conn will close at end of response     def _read_status(self):        # Initialize with Simple-Response defaults        line = self.fp.readline()        if self.debuglevel > 0:            print "reply:", repr(line)        if not line:            # Presumably, the server closed the connection before            # sending a valid response.            raise BadStatusLine(line)        try:            [version, status, reason] = line.split(None, 2)        except ValueError:            try:                [version, status] = line.split(None, 1)                reason = ""            except ValueError:                # empty version will cause next test to fail and status                # will be treated as 0.9 response.                version = ""        if not version.startswith('HTTP/'):            if self.strict:                self.close()                raise BadStatusLine(line)            else:                # assume it's a Simple-Response from an 0.9 server                self.fp = LineAndFileWrapper(line, self.fp)                return "HTTP/0.9", 200, ""         # The status code is a three-digit number        try:            status = int(status)            if status < 100 or status > 999:                raise BadStatusLine(line)        except ValueError:            raise BadStatusLine(line)        return version, status, reason     def begin(self):        if self.msg is not None:            # we've already started reading the response            return         # read until we get a non-100 response        while True:            version, status, reason = self._read_status()            if status != CONTINUE:                break            # skip the header from the 100 response            while True:                skip = self.fp.readline().strip()                if not skip:                    break                if self.debuglevel > 0:                    print "header:", skip         self.status = status        self.reason = reason.strip()        if version == 'HTTP/1.0':            self.version = 10        elif version.startswith('HTTP/1.'):            self.version = 11   # use HTTP/1.1 code for HTTP/1.x where x>=1        elif version == 'HTTP/0.9':            self.version = 9        else:            raise UnknownProtocol(version)         if self.version == 9:            self.length = None            self.chunked = 0            self.will_close = 1            self.msg = HTTPMessage(StringIO())            return         self.msg = HTTPMessage(self.fp, 0)        if self.debuglevel > 0:            for hdr in self.msg.headers:                print "header:", hdr,         # don't let the msg keep an fp        self.msg.fp = None         # are we using the chunked-style of transfer encoding?        tr_enc = self.msg.getheader('transfer-encoding')        if tr_enc and tr_enc.lower() == "chunked":            self.chunked = 1            self.chunk_left = None        else:            self.chunked = 0         # will the connection close at the end of the response?        self.will_close = self._check_close()         # do we have a Content-Length?        # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked"        length = self.msg.getheader('content-length')        if length and not self.chunked:            try:                self.length = int(length)            except ValueError:                self.length = None        else:            self.length = None         # does the body have a fixed length? (of zero)        if (status == NO_CONTENT or status == NOT_MODIFIED or            100 <= status < 200 or      # 1xx codes            self._method == 'HEAD'):            self.length = 0         # if the connection remains open, and we aren't using chunked, and        # a content-length was not provided, then assume that the connection        # WILL close.        if not self.will_close and \           not self.chunked and \           self.length is None:            self.will_close = 1     def _check_close(self):        conn = self.msg.getheader('connection')        if self.version == 11:            # An HTTP/1.1 proxy is assumed to stay open unless            # explicitly closed.            conn = self.msg.getheader('connection')            if conn and "close" in conn.lower():                return True            return False         # Some HTTP/1.0 implementations have support for persistent        # connections, using rules different than HTTP/1.1.         # For older HTTP, Keep-Alive indiciates persistent connection.        if self.msg.getheader('keep-alive'):            return False         # At least Akamai returns a "Connection: Keep-Alive" header,        # which was supposed to be sent by the client.        if conn and "keep-alive" in conn.lower():            return False         # Proxy-Connection is a netscape hack.        pconn = self.msg.getheader('proxy-connection')        if pconn and "keep-alive" in pconn.lower():            return False         # otherwise, assume it will close        return True     def close(self):        if self.fp:            self.fp.close()            self.fp = None     def isclosed(self):        # NOTE: it is possible that we will not ever call self.close(). This        #       case occurs when will_close is TRUE, length is None, and we        #       read up to the last byte, but NOT past it.        #        # IMPLIES: if will_close is FALSE, then self.close() will ALWAYS be        #          called, meaning self.isclosed() is meaningful.        return self.fp is None

Lines 1–500 of 1428. Load more