"""An extensible library for opening URLs using a variety of protocols The simplest way to use this module is to call the urlopen function,which accepts a string containing a URL or a Request object (describedbelow). It opens the URL and returns the results as file-likeobject; the returned object has some extra methods described below. The OpenerDirector manages a collection of Handler objects that doall the actual work. Each Handler implements a particular protocol oroption. The OpenerDirector is a composite object that invokes theHandlers needed to open the requested URL. For example, theHTTPHandler performs HTTP GET and POST requests and deals withnon-error returns. The HTTPRedirectHandler automatically deals withHTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandlerdeals with digest authentication. urlopen(url, data=None) -- basic usage is the same as originalurllib. pass the url and optionally data to post to an HTTP URL, andget a file-like object back. One difference is that you can also passa Request instance instead of URL. Raises a URLError (subclass ofIOError); for HTTP errors, raises an HTTPError, which can also betreated as a valid response. build_opener -- function that creates a new OpenerDirector instance.will install the default handlers. accepts one or more Handlers asarguments, either instances or Handler classes that it willinstantiate. if one of the argument is a subclass of the defaulthandler, the argument will be installed instead of the default. install_opener -- installs a new opener as the default opener. objects of interest:OpenerDirector -- Request -- an object that encapsulates the state of a request. thestate can be a simple as the URL. it can also include extra HTTPheaders, e.g. a User-Agent. BaseHandler -- exceptions:URLError-- a subclass of IOError, individual protocols have their ownspecific subclass HTTPError-- also a valid HTTP response, so you can treat an HTTP erroras an exceptional event or valid response internals:BaseHandler and parent_call_chain conventions Example usage: import urllib2 # set up authentication infoauthinfo = urllib2.HTTPBasicAuthHandler()authinfo.add_password('realm', 'host', 'username', 'password') proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"}) # build a new opener that adds authentication and caching FTP handlersopener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler) # install iturllib2.install_opener(opener) f = urllib2.urlopen('http://www.python.org/') """ # XXX issues:# If an authentication error handler that tries to perform# authentication for some reason but fails, how should the error be# signalled? The client needs to know the HTTP error code. But if# the handler knows that the problem was, e.g., that it didn't know# that hash algo that requested in the challenge, it would be good to# pass that information along to the client, too.# ftp errors aren't handled cleanly# check digest against correct (i.e. non-apache) implementation # Possible extensions:# complex proxies XXX not sure what exactly was meant by this# abstract factory for opener import base64import hashlibimport httplibimport mimetoolsimport osimport posixpathimport randomimport reimport socketimport sysimport timeimport urlparseimport bisect try: from cStringIO import StringIOexcept ImportError: from StringIO import StringIO from urllib import (unwrap, unquote, splittype, splithost, quote, addinfourl, splitport, splitgophertype, splitquery, splitattr, ftpwrapper, noheaders, splituser, splitpasswd, splitvalue) # support for FileHandler, proxies via environment variablesfrom urllib import localhost, url2pathname, getproxies # used in User-Agent header sent__version__ = sys.version[:3] _opener = Nonedef urlopen(url, data=None): global _opener if _opener is None: _opener = build_opener() return _opener.open(url, data) def install_opener(opener): global _opener _opener = opener # do these error classes make sense?# make sure all of the IOError stuff is overridden. we just want to be# subtypes. class URLError(IOError): # URLError is a sub-type of IOError, but it doesn't share any of # the implementation. need to override __init__ and __str__. # It sets self.args for compatibility with other EnvironmentError # subclasses, but args doesn't have the typical format with errno in # slot 0 and strerror in slot 1. This may be better than nothing. def __init__(self, reason): self.args = reason, self.reason = reason def __str__(self): return '<urlopen error %s>' % self.reason class HTTPError(URLError, addinfourl): """Raised when HTTP error occurs, but also acts like non-error return""" __super_init = addinfourl.__init__ def __init__(self, url, code, msg, hdrs, fp): self.code = code self.msg = msg self.hdrs = hdrs self.fp = fp self.filename = url # The addinfourl classes depend on fp being a valid file # object. In some cases, the HTTPError may not have a valid # file object. If this happens, the simplest workaround is to # not initialize the base classes. if fp is not None: self.__super_init(fp, hdrs, url) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) class GopherError(URLError): pass # copied from cookielib.py_cut_port_re = re.compile(r":\d+$")def request_host(request): """Return request-host, as defined by RFC 2965. Variation from RFC: returned value is lowercased, for convenient comparison. """ url = request.get_full_url() host = urlparse.urlparse(url)[1] if host == "": host = request.get_header("Host", "") # remove port, if present host = _cut_port_re.sub("", host, 1) return host.lower() class Request: def __init__(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False): # unwrap('<URL:type://host/path>') --> 'type://host/path' self.__original = unwrap(url) self.type = None # self.__r_type is what's left after doing the splittype self.host = None self.port = None self.data = data self.headers = {} for key, value in headers.items(): self.add_header(key, value) self.unredirected_hdrs = {} if origin_req_host is None: origin_req_host = request_host(self) self.origin_req_host = origin_req_host self.unverifiable = unverifiable def __getattr__(self, attr): # XXX this is a fallback mechanism to guard against these # methods getting called in a non-standard order. this may be # too complicated and/or unnecessary. # XXX should the __r_XXX attributes be public? if attr[:12] == '_Request__r_': name = attr[12:] if hasattr(Request, 'get_' + name): getattr(self, 'get_' + name)() return getattr(self, attr) raise AttributeError, attr def get_method(self): if self.has_data(): return "POST" else: return "GET" # XXX these helper methods are lame def add_data(self, data): self.data = data def has_data(self): return self.data is not None def get_data(self): return self.data def get_full_url(self): return self.__original def get_type(self): if self.type is None: self.type, self.__r_type = splittype(self.__original) if self.type is None: raise ValueError, "unknown url type: %s" % self.__original return self.type def get_host(self): if self.host is None: self.host, self.__r_host = splithost(self.__r_type) if self.host: self.host = unquote(self.host) return self.host def get_selector(self): return self.__r_host def set_proxy(self, host, type): self.host, self.type = host, type self.__r_host = self.__original def get_origin_req_host(self): return self.origin_req_host def is_unverifiable(self): return self.unverifiable def add_header(self, key, val): # useful for something like authentication self.headers[key.capitalize()] = val def add_unredirected_header(self, key, val): # will not be added to a redirected request self.unredirected_hdrs[key.capitalize()] = val def has_header(self, header_name): return (header_name in self.headers or header_name in self.unredirected_hdrs) def get_header(self, header_name, default=None): return self.headers.get( header_name, self.unredirected_hdrs.get(header_name, default)) def header_items(self): hdrs = self.unredirected_hdrs.copy() hdrs.update(self.headers) return hdrs.items() class OpenerDirector: def __init__(self): client_version = "Python-urllib/%s" % __version__ self.addheaders = [('User-agent', client_version)] # manage the individual handlers self.handlers = [] self.handle_open = {} self.handle_error = {} self.process_response = {} self.process_request = {} def add_handler(self, handler): added = False for meth in dir(handler): if meth in ["redirect_request", "do_open", "proxy_open"]: # oops, coincidental match continue i = meth.find("_") protocol = meth[:i] condition = meth[i+1:] if condition.startswith("error"): j = condition.find("_") + i + 1 kind = meth[j+1:] try: kind = int(kind) except ValueError: pass lookup = self.handle_error.get(protocol, {}) self.handle_error[protocol] = lookup elif condition == "open": kind = protocol lookup = self.handle_open elif condition == "response": kind = protocol lookup = self.process_response elif condition == "request": kind = protocol lookup = self.process_request else: continue handlers = lookup.setdefault(kind, []) if handlers: bisect.insort(handlers, handler) else: handlers.append(handler) added = True if added: # XXX why does self.handlers need to be sorted? bisect.insort(self.handlers, handler) handler.add_parent(self) def close(self): # Only exists for backwards compatibility. pass def _call_chain(self, chain, kind, meth_name, *args): # Handlers raise an exception if no one else should try to handle # the request, or return None if they can't but another handler # could. Otherwise, they return the response. handlers = chain.get(kind, ()) for handler in handlers: func = getattr(handler, meth_name) result = func(*args) if result is not None: return result def open(self, fullurl, data=None): # accept a URL or a Request object if isinstance(fullurl, basestring): req = Request(fullurl, data) else: req = fullurl if data is not None: req.add_data(data) protocol = req.get_type() # pre-process request meth_name = protocol+"_request" for processor in self.process_request.get(protocol, []): meth = getattr(processor, meth_name) req = meth(req) response = self._open(req, data) # post-process response meth_name = protocol+"_response" for processor in self.process_response.get(protocol, []): meth = getattr(processor, meth_name) response = meth(req, response) return response def _open(self, req, data=None): result = self._call_chain(self.handle_open, 'default', 'default_open', req) if result: return result protocol = req.get_type() result = self._call_chain(self.handle_open, protocol, protocol + '_open', req) if result: return result return self._call_chain(self.handle_open, 'unknown', 'unknown_open', req) def error(self, proto, *args): if proto in ('http', 'https'): # XXX http[s] protocols are special-cased dict = self.handle_error['http'] # https is not different than http proto = args[2] # YUCK! meth_name = 'http_error_%s' % proto http_err = 1 orig_args = args else: dict = self.handle_error meth_name = proto + '_error' http_err = 0 args = (dict, proto, meth_name) + args result = self._call_chain(*args) if result: return result if http_err: args = (dict, 'default', 'http_error_default') + orig_args return self._call_chain(*args) # XXX probably also want an abstract factory that knows when it makes# sense to skip a superclass in favor of a subclass and when it might# make sense to include both def build_opener(*handlers): """Create an opener object from a list of handlers. The opener will use several default handlers, including support for HTTP and FTP. If any of the handlers passed as arguments are subclasses of the default handlers, the default handlers will not be used. """ import types def isclass(obj): return isinstance(obj, types.ClassType) or hasattr(obj, "__bases__") opener = OpenerDirector() default_classes = [ProxyHandler, UnknownHandler, HTTPHandler, HTTPDefaultErrorHandler, HTTPRedirectHandler, FTPHandler, FileHandler, HTTPErrorProcessor] if hasattr(httplib, 'HTTPS'): default_classes.append(HTTPSHandler) skip = [] for klass in default_classes: for check in handlers: if isclass(check): if issubclass(check, klass): skip.append(klass) elif isinstance(check, klass): skip.append(klass) for klass in skip: default_classes.remove(klass) for klass in default_classes: opener.add_handler(klass()) for h in handlers: if isclass(h): h = h() opener.add_handler(h) return opener class BaseHandler: handler_order = 500 def add_parent(self, parent): self.parent = parent def close(self): # Only exists for backwards compatibility pass def __lt__(self, other): if not hasattr(other, "handler_order"): # Try to preserve the old behavior of having custom classes # inserted after default ones (works only for custom user # classes which are not aware of handler_order). return True return self.handler_order < other.handler_order class HTTPErrorProcessor(BaseHandler): """Process HTTP error responses.""" handler_order = 1000 # after all other processing def http_response(self, request, response): code, msg, hdrs = response.code, response.msg, response.info() if code not in (200, 206): response = self.parent.error( 'http', request, response, code, msg, hdrs) return response https_response = http_response class HTTPDefaultErrorHandler(BaseHandler): def http_error_default(self, req, fp, code, msg, hdrs): raise HTTPError(req.get_full_url(), code, msg, hdrs, fp) Lines 1–500 of 1353. Load more
"""An extensible library for opening URLs using a variety of protocols The simplest way to use this module is to call the urlopen function, which accepts a string containing a URL or a Request object (described below). It opens the URL and returns the results as file-like object; the returned object has some extra methods described below. The OpenerDirector manages a collection of Handler objects that do all the actual work. Each Handler implements a particular protocol or option. The OpenerDirector is a composite object that invokes the Handlers needed to open the requested URL. For example, the HTTPHandler performs HTTP GET and POST requests and deals with non-error returns. The HTTPRedirectHandler automatically deals with HTTP 301, 302, 303 and 307 redirect errors, and the HTTPDigestAuthHandler deals with digest authentication. urlopen(url, data=None) -- basic usage is the same as original urllib. pass the url and optionally data to post to an HTTP URL, and get a file-like object back. One difference is that you can also pass a Request instance instead of URL. Raises a URLError (subclass of IOError); for HTTP errors, raises an HTTPError, which can also be treated as a valid response. build_opener -- function that creates a new OpenerDirector instance. will install the default handlers. accepts one or more Handlers as arguments, either instances or Handler classes that it will instantiate. if one of the argument is a subclass of the default handler, the argument will be installed instead of the default. install_opener -- installs a new opener as the default opener. objects of interest: OpenerDirector -- Request -- an object that encapsulates the state of a request. the state can be a simple as the URL. it can also include extra HTTP headers, e.g. a User-Agent. BaseHandler -- exceptions: URLError-- a subclass of IOError, individual protocols have their own specific subclass HTTPError-- also a valid HTTP response, so you can treat an HTTP error as an exceptional event or valid response internals: BaseHandler and parent _call_chain conventions Example usage: import urllib2 # set up authentication info authinfo = urllib2.HTTPBasicAuthHandler() authinfo.add_password('realm', 'host', 'username', 'password') proxy_support = urllib2.ProxyHandler({"http" : "http://ahad-haam:3128"}) # build a new opener that adds authentication and caching FTP handlers opener = urllib2.build_opener(proxy_support, authinfo, urllib2.CacheFTPHandler) # install it urllib2.install_opener(opener) f = urllib2.urlopen('http://www.python.org/') """ # XXX issues: # If an authentication error handler that tries to perform # authentication for some reason but fails, how should the error be # signalled? The client needs to know the HTTP error code. But if # the handler knows that the problem was, e.g., that it didn't know # that hash algo that requested in the challenge, it would be good to # pass that information along to the client, too. # ftp errors aren't handled cleanly # check digest against correct (i.e. non-apache) implementation # Possible extensions: # complex proxies XXX not sure what exactly was meant by this # abstract factory for opener import base64 import hashlib import httplib import mimetools import os import posixpath import random import re import socket import sys import time import urlparse import bisect try: from cStringIO import StringIO except ImportError: from StringIO import StringIO from urllib import (unwrap, unquote, splittype, splithost, quote, addinfourl, splitport, splitgophertype, splitquery, splitattr, ftpwrapper, noheaders, splituser, splitpasswd, splitvalue) # support for FileHandler, proxies via environment variables from urllib import localhost, url2pathname, getproxies # used in User-Agent header sent __version__ = sys.version[:3] _opener = None def urlopen(url, data=None): global _opener if _opener is None: _opener = build_opener() return _opener.open(url, data) def install_opener(opener): global _opener _opener = opener # do these error classes make sense? # make sure all of the IOError stuff is overridden. we just want to be # subtypes. class URLError(IOError): # URLError is a sub-type of IOError, but it doesn't share any of # the implementation. need to override __init__ and __str__. # It sets self.args for compatibility with other EnvironmentError # subclasses, but args doesn't have the typical format with errno in # slot 0 and strerror in slot 1. This may be better than nothing. def __init__(self, reason): self.args = reason, self.reason = reason def __str__(self): return '<urlopen error %s>' % self.reason class HTTPError(URLError, addinfourl): """Raised when HTTP error occurs, but also acts like non-error return""" __super_init = addinfourl.__init__ def __init__(self, url, code, msg, hdrs, fp): self.code = code self.msg = msg self.hdrs = hdrs self.fp = fp self.filename = url # The addinfourl classes depend on fp being a valid file # object. In some cases, the HTTPError may not have a valid # file object. If this happens, the simplest workaround is to # not initialize the base classes. if fp is not None: self.__super_init(fp, hdrs, url) def __str__(self): return 'HTTP Error %s: %s' % (self.code, self.msg) class GopherError(URLError): pass # copied from cookielib.py _cut_port_re = re.compile(r":\d+$") def request_host(request): """Return request-host, as defined by RFC 2965. Variation from RFC: returned value is lowercased, for convenient comparison. """ url = request.get_full_url() host = urlparse.urlparse(url)[1] if host == "": host = request.get_header("Host", "") # remove port, if present host = _cut_port_re.sub("", host, 1) return host.lower() class Request: def __init__(self, url, data=None, headers={}, origin_req_host=None, unverifiable=False): # unwrap('<URL:type://host/path>') --> 'type://host/path' self.__original = unwrap(url) self.type = None # self.__r_type is what's left after doing the splittype self.host = None self.port = None self.data = data self.headers = {} for key, value in headers.items(): self.add_header(key, value) self.unredirected_hdrs = {} if origin_req_host is None: origin_req_host = request_host(self) self.origin_req_host = origin_req_host self.unverifiable = unverifiable def __getattr__(self, attr): # XXX this is a fallback mechanism to guard against these # methods getting called in a non-standard order. this may be # too complicated and/or unnecessary. # XXX should the __r_XXX attributes be public? if attr[:12] == '_Request__r_': name = attr[12:] if hasattr(Request, 'get_' + name): getattr(self, 'get_' + name)() return getattr(self, attr) raise AttributeError, attr def get_method(self): if self.has_data(): return "POST" else: return "GET" # XXX these helper methods are lame def add_data(self, data): self.data = data def has_data(self): return self.data is not None def get_data(self): return self.data def get_full_url(self): return self.__original def get_type(self): if self.type is None: self.type, self.__r_type = splittype(self.__original) if self.type is None: raise ValueError, "unknown url type: %s" % self.__original return self.type def get_host(self): if self.host is None: self.host, self.__r_host = splithost(self.__r_type) if self.host: self.host = unquote(self.host) return self.host def get_selector(self): return self.__r_host def set_proxy(self, host, type): self.host, self.type = host, type self.__r_host = self.__original def get_origin_req_host(self): return self.origin_req_host def is_unverifiable(self): return self.unverifiable def add_header(self, key, val): # useful for something like authentication self.headers[key.capitalize()] = val def add_unredirected_header(self, key, val): # will not be added to a redirected request self.unredirected_hdrs[key.capitalize()] = val def has_header(self, header_name): return (header_name in self.headers or header_name in self.unredirected_hdrs) def get_header(self, header_name, default=None): return self.headers.get( header_name, self.unredirected_hdrs.get(header_name, default)) def header_items(self): hdrs = self.unredirected_hdrs.copy() hdrs.update(self.headers) return hdrs.items() class OpenerDirector: def __init__(self): client_version = "Python-urllib/%s" % __version__ self.addheaders = [('User-agent', client_version)] # manage the individual handlers self.handlers = [] self.handle_open = {} self.handle_error = {} self.process_response = {} self.process_request = {} def add_handler(self, handler): added = False for meth in dir(handler): if meth in ["redirect_request", "do_open", "proxy_open"]: # oops, coincidental match continue i = meth.find("_") protocol = meth[:i] condition = meth[i+1:] if condition.startswith("error"): j = condition.find("_") + i + 1 kind = meth[j+1:] try: kind = int(kind) except ValueError: pass lookup = self.handle_error.get(protocol, {}) self.handle_error[protocol] = lookup elif condition == "open": kind = protocol lookup = self.handle_open elif condition == "response": kind = protocol lookup = self.process_response elif condition == "request": kind = protocol lookup = self.process_request else: continue handlers = lookup.setdefault(kind, []) if handlers: bisect.insort(handlers, handler) else: handlers.append(handler) added = True if added: # XXX why does self.handlers need to be sorted? bisect.insort(self.handlers, handler) handler.add_parent(self) def close(self): # Only exists for backwards compatibility. pass def _call_chain(self, chain, kind, meth_name, *args): # Handlers raise an exception if no one else should try to handle # the request, or return None if they can't but another handler # could. Otherwise, they return the response. handlers = chain.get(kind, ()) for handler in handlers: func = getattr(handler, meth_name) result = func(*args) if result is not None: return result def open(self, fullurl, data=None): # accept a URL or a Request object if isinstance(fullurl, basestring): req = Request(fullurl, data) else: req = fullurl if data is not None: req.add_data(data) protocol = req.get_type() # pre-process request meth_name = protocol+"_request" for processor in self.process_request.get(protocol, []): meth = getattr(processor, meth_name) req = meth(req) response = self._open(req, data) # post-process response meth_name = protocol+"_response" for processor in self.process_response.get(protocol, []): meth = getattr(processor, meth_name) response = meth(req, response) return response def _open(self, req, data=None): result = self._call_chain(self.handle_open, 'default', 'default_open', req) if result: return result protocol = req.get_type() result = self._call_chain(self.handle_open, protocol, protocol + '_open', req) if result: return result return self._call_chain(self.handle_open, 'unknown', 'unknown_open', req) def error(self, proto, *args): if proto in ('http', 'https'): # XXX http[s] protocols are special-cased dict = self.handle_error['http'] # https is not different than http proto = args[2] # YUCK! meth_name = 'http_error_%s' % proto http_err = 1 orig_args = args else: dict = self.handle_error meth_name = proto + '_error' http_err = 0 args = (dict, proto, meth_name) + args result = self._call_chain(*args) if result: return result if http_err: args = (dict, 'default', 'http_error_default') + orig_args return self._call_chain(*args) # XXX probably also want an abstract factory that knows when it makes # sense to skip a superclass in favor of a subclass and when it might # make sense to include both def build_opener(*handlers): """Create an opener object from a list of handlers. The opener will use several default handlers, including support for HTTP and FTP. If any of the handlers passed as arguments are subclasses of the default handlers, the default handlers will not be used. """ import types def isclass(obj): return isinstance(obj, types.ClassType) or hasattr(obj, "__bases__") opener = OpenerDirector() default_classes = [ProxyHandler, UnknownHandler, HTTPHandler, HTTPDefaultErrorHandler, HTTPRedirectHandler, FTPHandler, FileHandler, HTTPErrorProcessor] if hasattr(httplib, 'HTTPS'): default_classes.append(HTTPSHandler) skip = [] for klass in default_classes: for check in handlers: if isclass(check): if issubclass(check, klass): skip.append(klass) elif isinstance(check, klass): skip.append(klass) for klass in skip: default_classes.remove(klass) for klass in default_classes: opener.add_handler(klass()) for h in handlers: if isclass(h): h = h() opener.add_handler(h) return opener class BaseHandler: handler_order = 500 def add_parent(self, parent): self.parent = parent def close(self): # Only exists for backwards compatibility pass def __lt__(self, other): if not hasattr(other, "handler_order"): # Try to preserve the old behavior of having custom classes # inserted after default ones (works only for custom user # classes which are not aware of handler_order). return True return self.handler_order < other.handler_order class HTTPErrorProcessor(BaseHandler): """Process HTTP error responses.""" handler_order = 1000 # after all other processing def http_response(self, request, response): code, msg, hdrs = response.code, response.msg, response.info() if code not in (200, 206): response = self.parent.error( 'http', request, response, code, msg, hdrs) return response https_response = http_response class HTTPDefaultErrorHandler(BaseHandler): def http_error_default(self, req, fp, code, msg, hdrs): raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)