Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(214)

Side by Side Diff: python2/httplib2/__init__.py

Issue 6506074: Making httplib2.Http instances pickleable. (Closed)
Patch Set: Making sure connections gets dropped and making tests less brittle. Created 1 year, 7 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | python2/httplib2test.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 from __future__ import generators 1 from __future__ import generators
2 """ 2 """
3 httplib2 3 httplib2
4 4
5 A caching http interface that supports ETags and gzip 5 A caching http interface that supports ETags and gzip
6 to conserve bandwidth. 6 to conserve bandwidth.
7 7
8 Requires Python 2.3 or later 8 Requires Python 2.3 or later
9 9
10 Changelog: 10 Changelog:
(...skipping 745 matching lines...) Expand 10 before | Expand all | Expand 10 after
756 self.proxy_user = proxy_user 756 self.proxy_user = proxy_user
757 self.proxy_pass = proxy_pass 757 self.proxy_pass = proxy_pass
758 758
759 def astuple(self): 759 def astuple(self):
760 return (self.proxy_type, self.proxy_host, self.proxy_port, 760 return (self.proxy_type, self.proxy_host, self.proxy_port,
761 self.proxy_rdns, self.proxy_user, self.proxy_pass) 761 self.proxy_rdns, self.proxy_user, self.proxy_pass)
762 762
763 def isgood(self): 763 def isgood(self):
764 return (self.proxy_host != None) and (self.proxy_port != None) 764 return (self.proxy_host != None) and (self.proxy_port != None)
765 765
766 @classmethod
767 def from_environment(cls, method='http'):
768 """
769 Read proxy info from the environment variables.
770 """
771 if method not in ['http', 'https']:
772 return
773
774 env_var = method + '_proxy'
775 url = os.environ.get(env_var, os.environ.get(env_var.upper()))
776 if not url:
777 return
778 pi = cls.from_url(url, method)
779
780 no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', ''))
781 bypass_hosts = []
782 if no_proxy:
783 bypass_hosts = no_proxy.split(',')
784 # special case, no_proxy=* means all hosts bypassed
785 if no_proxy == '*':
786 bypass_hosts = AllHosts
787
788 pi.bypass_hosts = bypass_hosts
789 return pi
790
791 @classmethod
792 def from_url(cls, url, method='http'):
793 """
794 Construct a ProxyInfo from a URL (such as http_proxy env var)
795 """
796 url = urlparse.urlparse(url)
797 username = None
798 password = None
799 port = None
800 if '@' in url[1]:
801 ident, host_port = url[1].split('@', 1)
802 if ':' in ident:
803 username, password = ident.split(':', 1)
804 else:
805 password = ident
806 else:
807 host_port = url[1]
808 if ':' in host_port:
809 host, port = host_port.split(':', 1)
810 else:
811 host = host_port
812
813 if port:
814 port = int(port)
815 else:
816 port = dict(https=443, http=80)[method]
817
818 proxy_type = 3 # socks.PROXY_TYPE_HTTP
819 return cls(
820 proxy_type = proxy_type,
821 proxy_host = host,
822 proxy_port = port,
823 proxy_user = username or None,
824 proxy_pass = password or None,
825 )
826
827 def applies_to(self, hostname): 766 def applies_to(self, hostname):
828 return not self.bypass_host(hostname) 767 return not self.bypass_host(hostname)
829 768
830 def bypass_host(self, hostname): 769 def bypass_host(self, hostname):
831 """Has this host been excluded from the proxy config""" 770 """Has this host been excluded from the proxy config"""
832 if self.bypass_hosts is AllHosts: 771 if self.bypass_hosts is AllHosts:
833 return True 772 return True
834 773
835 bypass = False 774 bypass = False
836 for domain in self.bypass_hosts: 775 for domain in self.bypass_hosts:
837 if hostname.endswith(domain): 776 if hostname.endswith(domain):
838 bypass = True 777 bypass = True
839 778
840 return bypass 779 return bypass
841 780
842 781
782 def proxy_info_from_environment(method='http'):
783 """
784 Read proxy info from the environment variables.
785 """
786 if method not in ['http', 'https']:
787 return
788
789 env_var = method + '_proxy'
790 url = os.environ.get(env_var, os.environ.get(env_var.upper()))
791 if not url:
792 return
793 pi = proxy_info_from_url(url, method)
794
795 no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', ''))
796 bypass_hosts = []
797 if no_proxy:
798 bypass_hosts = no_proxy.split(',')
799 # special case, no_proxy=* means all hosts bypassed
800 if no_proxy == '*':
801 bypass_hosts = AllHosts
802
803 pi.bypass_hosts = bypass_hosts
804 return pi
805
806 def proxy_info_from_url(url, method='http'):
807 """
808 Construct a ProxyInfo from a URL (such as http_proxy env var)
809 """
810 url = urlparse.urlparse(url)
811 username = None
812 password = None
813 port = None
814 if '@' in url[1]:
815 ident, host_port = url[1].split('@', 1)
816 if ':' in ident:
817 username, password = ident.split(':', 1)
818 else:
819 password = ident
820 else:
821 host_port = url[1]
822 if ':' in host_port:
823 host, port = host_port.split(':', 1)
824 else:
825 host = host_port
826
827 if port:
828 port = int(port)
829 else:
830 port = dict(https=443, http=80)[method]
831
832 proxy_type = 3 # socks.PROXY_TYPE_HTTP
833 return ProxyInfo(
834 proxy_type = proxy_type,
835 proxy_host = host,
836 proxy_port = port,
837 proxy_user = username or None,
838 proxy_pass = password or None,
839 )
840
841
843 class HTTPConnectionWithTimeout(httplib.HTTPConnection): 842 class HTTPConnectionWithTimeout(httplib.HTTPConnection):
844 """ 843 """
845 HTTPConnection subclass that supports timeouts 844 HTTPConnection subclass that supports timeouts
846 845
847 All timeouts are in seconds. If None is passed for timeout then 846 All timeouts are in seconds. If None is passed for timeout then
848 Python's default timeout for sockets will be used. See for example 847 Python's default timeout for sockets will be used. See for example
849 the docs of socket.setdefaulttimeout(): 848 the docs of socket.setdefaulttimeout():
850 http://docs.python.org/library/socket.html#socket.setdefaulttimeout 849 http://docs.python.org/library/socket.html#socket.setdefaulttimeout
851 """ 850 """
852 851
(...skipping 210 matching lines...) Expand 10 before | Expand all | Expand 10 after
1063 if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: 1062 if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None:
1064 raise ImportError # Bail out; we're not actually running on App Engine. 1063 raise ImportError # Bail out; we're not actually running on App Engine.
1065 from google.appengine.api.urlfetch import fetch 1064 from google.appengine.api.urlfetch import fetch
1066 from google.appengine.api.urlfetch import InvalidURLError 1065 from google.appengine.api.urlfetch import InvalidURLError
1067 from google.appengine.api.urlfetch import DownloadError 1066 from google.appengine.api.urlfetch import DownloadError
1068 from google.appengine.api.urlfetch import ResponseTooLargeError 1067 from google.appengine.api.urlfetch import ResponseTooLargeError
1069 from google.appengine.api.urlfetch import SSLCertificateError 1068 from google.appengine.api.urlfetch import SSLCertificateError
1070 1069
1071 1070
1072 class ResponseDict(dict): 1071 class ResponseDict(dict):
1073 """Is a dictionary that also has a read() method, so 1072 """Dictionary with a read() method; can pass off as httplib.HTTPResponse."""
1074 that it can pass itself off as an httlib.HTTPResponse().""" 1073 def __init__(self):
1074 self.content = kwargs.pop('content', None)
1075 return super(ResponseDict, self).__init__(*args, **kwargs)
jcgregorio_google 2012/09/13 16:12:13 Where do *args, and **kwargs come from?
dhermes 2012/09/13 16:39:51 Good catch. I must have accidentally left it out w
1076
1075 def read(self): 1077 def read(self):
1076 pass 1078 return self.content
1077 1079
1078 1080
1079 class AppEngineHttpConnection(object): 1081 class AppEngineHttpConnection(object):
1080 """Emulates an httplib.HTTPConnection object, but actually uses the Google 1082 """Emulates an httplib.HTTPConnection object, but actually uses the Google
1081 App Engine urlfetch library. This allows the timeout to be properly used on 1083 App Engine urlfetch library. This allows the timeout to be properly used on
1082 Google App Engine, and avoids using httplib, which on Google App Engine is 1084 Google App Engine, and avoids using httplib, which on Google App Engine is
1083 just another wrapper around urlfetch. 1085 just another wrapper around urlfetch.
1084 """ 1086 """
1085 def __init__(self, host, port=None, key_file=None, cert_file=None, 1087 def __init__(self, host, port=None, key_file=None, cert_file=None,
1086 strict=None, timeout=None, proxy_info=None, ca_certs=None, 1088 strict=None, timeout=None, proxy_info=None, ca_certs=None,
(...skipping 16 matching lines...) Expand all
1103 absolute_uri = '%s://%s%s' % (self.scheme, netloc, url) 1105 absolute_uri = '%s://%s%s' % (self.scheme, netloc, url)
1104 try: 1106 try:
1105 try: # 'body' can be a stream. 1107 try: # 'body' can be a stream.
1106 body = body.read() 1108 body = body.read()
1107 except AttributeError: 1109 except AttributeError:
1108 pass 1110 pass
1109 response = fetch(absolute_uri, payload=body, method=method, 1111 response = fetch(absolute_uri, payload=body, method=method,
1110 headers=headers, allow_truncated=False, follow_redirects=False, 1112 headers=headers, allow_truncated=False, follow_redirects=False,
1111 deadline=self.timeout, 1113 deadline=self.timeout,
1112 validate_certificate=self.validate_certificate) 1114 validate_certificate=self.validate_certificate)
1113 self.response = ResponseDict(response.headers) 1115 self.response = ResponseDict(response.headers, response.content)
1114 self.response['status'] = str(response.status_code) 1116 self.response['status'] = str(response.status_code)
1115 self.response['reason'] = httplib.responses.get(response.status_code, 'O k') 1117 self.response['reason'] = httplib.responses.get(response.status_code, 'O k')
1116 self.response.status = response.status_code 1118 self.response.status = response.status_code
1117 setattr(self.response, 'read', lambda : response.content)
1118 1119
1119 # Make sure the exceptions raised match the exceptions expected. 1120 # Make sure the exceptions raised match the exceptions expected.
1120 except InvalidURLError: 1121 except InvalidURLError:
1121 raise socket.gaierror('') 1122 raise socket.gaierror('')
1122 except (DownloadError, ResponseTooLargeError, SSLCertificateError): 1123 except (DownloadError, ResponseTooLargeError, SSLCertificateError):
1123 raise httplib.HTTPException() 1124 raise httplib.HTTPException()
1124 1125
1125 def getresponse(self): 1126 def getresponse(self):
1126 if self.response: 1127 if self.response:
1127 return self.response 1128 return self.response
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after
1164 - ETags 1165 - ETags
1165 - compression, 1166 - compression,
1166 - HTTPS 1167 - HTTPS
1167 - Basic 1168 - Basic
1168 - Digest 1169 - Digest
1169 - WSSE 1170 - WSSE
1170 1171
1171 and more. 1172 and more.
1172 """ 1173 """
1173 def __init__(self, cache=None, timeout=None, 1174 def __init__(self, cache=None, timeout=None,
1174 proxy_info=ProxyInfo.from_environment, 1175 proxy_info=proxy_info_from_environment,
1175 ca_certs=None, disable_ssl_certificate_validation=False): 1176 ca_certs=None, disable_ssl_certificate_validation=False):
1176 """If 'cache' is a string then it is used as a directory name for 1177 """If 'cache' is a string then it is used as a directory name for
1177 a disk cache. Otherwise it must be an object that supports the 1178 a disk cache. Otherwise it must be an object that supports the
1178 same interface as FileCache. 1179 same interface as FileCache.
1179 1180
1180 All timeouts are in seconds. If None is passed for timeout 1181 All timeouts are in seconds. If None is passed for timeout
1181 then Python's default timeout for sockets will be used. See 1182 then Python's default timeout for sockets will be used. See
1182 for example the docs of socket.setdefaulttimeout(): 1183 for example the docs of socket.setdefaulttimeout():
1183 http://docs.python.org/library/socket.html#socket.setdefaulttimeout 1184 http://docs.python.org/library/socket.html#socket.setdefaulttimeout
1184 1185
1185 `proxy_info` may be: 1186 `proxy_info` may be:
1186 - a callable that takes the http scheme ('http' or 'https') and 1187 - a callable that takes the http scheme ('http' or 'https') and
1187 returns a ProxyInfo instance per request. By default, uses 1188 returns a ProxyInfo instance per request. By default, uses
1188 ProxyInfo.from_environment. 1189 proxy_nfo_from_environment.
1189 - a ProxyInfo instance (static proxy config). 1190 - a ProxyInfo instance (static proxy config).
1190 - None (proxy disabled). 1191 - None (proxy disabled).
1191 1192
1192 ca_certs is the path of a file containing root CA certificates for SSL 1193 ca_certs is the path of a file containing root CA certificates for SSL
1193 server certificate validation. By default, a CA cert file bundled with 1194 server certificate validation. By default, a CA cert file bundled with
1194 httplib2 is used. 1195 httplib2 is used.
1195 1196
1196 If disable_ssl_certificate_validation is true, SSL cert validation will 1197 If disable_ssl_certificate_validation is true, SSL cert validation will
1197 not be performed. 1198 not be performed.
1198 """ 1199 """
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after
1232 1233
1233 self.ignore_etag = False 1234 self.ignore_etag = False
1234 1235
1235 self.force_exception_to_status_code = False 1236 self.force_exception_to_status_code = False
1236 1237
1237 self.timeout = timeout 1238 self.timeout = timeout
1238 1239
1239 # Keep Authorization: headers on a redirect. 1240 # Keep Authorization: headers on a redirect.
1240 self.forward_authorization_headers = False 1241 self.forward_authorization_headers = False
1241 1242
1243 def __getstate__(self):
1244 state_dict = copy.copy(self.__dict__)
1245 # In case request is augmented by some foreign object such as
1246 # credentials which handle auth
1247 if 'request' in state_dict:
1248 del state_dict['request']
1249 if 'connections' in state_dict:
1250 del state_dict['connections']
1251 return state_dict
1252
1253 def __setstate__(self, state):
1254 self.__dict__.update(state)
1255 self.connections = {}
1256
1242 def _auth_from_challenge(self, host, request_uri, headers, response, content ): 1257 def _auth_from_challenge(self, host, request_uri, headers, response, content ):
1243 """A generator that creates Authorization objects 1258 """A generator that creates Authorization objects
1244 that can be applied to requests. 1259 that can be applied to requests.
1245 """ 1260 """
1246 challenges = _parse_www_authenticate(response, 'www-authenticate') 1261 challenges = _parse_www_authenticate(response, 'www-authenticate')
1247 for cred in self.credentials.iter(host): 1262 for cred in self.credentials.iter(host):
1248 for scheme in AUTH_SCHEME_ORDER: 1263 for scheme in AUTH_SCHEME_ORDER:
1249 if challenges.has_key(scheme): 1264 if challenges.has_key(scheme):
1250 yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, h eaders, response, content, self) 1265 yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, h eaders, response, content, self)
1251 1266
(...skipping 414 matching lines...) Expand 10 before | Expand all | Expand 10 after
1666 self[key.lower()] = value 1681 self[key.lower()] = value
1667 self.status = int(self.get('status', self.status)) 1682 self.status = int(self.get('status', self.status))
1668 self.reason = self.get('reason', self.reason) 1683 self.reason = self.get('reason', self.reason)
1669 1684
1670 1685
1671 def __getattr__(self, name): 1686 def __getattr__(self, name):
1672 if name == 'dict': 1687 if name == 'dict':
1673 return self 1688 return self
1674 else: 1689 else:
1675 raise AttributeError, name 1690 raise AttributeError, name
OLDNEW
« no previous file with comments | « no previous file | python2/httplib2test.py » ('j') | no next file with comments »

Powered by Google App Engine
RSS Feeds Recent Issues | This issue
This is Rietveld 1278:e6ce13d99bf5