Left: | ||
Right: |
OLD | NEW |
---|---|
1 from __future__ import generators | 1 from __future__ import generators |
2 """ | 2 """ |
3 httplib2 | 3 httplib2 |
4 | 4 |
5 A caching http interface that supports ETags and gzip | 5 A caching http interface that supports ETags and gzip |
6 to conserve bandwidth. | 6 to conserve bandwidth. |
7 | 7 |
8 Requires Python 2.3 or later | 8 Requires Python 2.3 or later |
9 | 9 |
10 Changelog: | 10 Changelog: |
(...skipping 745 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
756 self.proxy_user = proxy_user | 756 self.proxy_user = proxy_user |
757 self.proxy_pass = proxy_pass | 757 self.proxy_pass = proxy_pass |
758 | 758 |
759 def astuple(self): | 759 def astuple(self): |
760 return (self.proxy_type, self.proxy_host, self.proxy_port, | 760 return (self.proxy_type, self.proxy_host, self.proxy_port, |
761 self.proxy_rdns, self.proxy_user, self.proxy_pass) | 761 self.proxy_rdns, self.proxy_user, self.proxy_pass) |
762 | 762 |
763 def isgood(self): | 763 def isgood(self): |
764 return (self.proxy_host != None) and (self.proxy_port != None) | 764 return (self.proxy_host != None) and (self.proxy_port != None) |
765 | 765 |
766 @classmethod | |
767 def from_environment(cls, method='http'): | |
768 """ | |
769 Read proxy info from the environment variables. | |
770 """ | |
771 if method not in ['http', 'https']: | |
772 return | |
773 | |
774 env_var = method + '_proxy' | |
775 url = os.environ.get(env_var, os.environ.get(env_var.upper())) | |
776 if not url: | |
777 return | |
778 pi = cls.from_url(url, method) | |
779 | |
780 no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')) | |
781 bypass_hosts = [] | |
782 if no_proxy: | |
783 bypass_hosts = no_proxy.split(',') | |
784 # special case, no_proxy=* means all hosts bypassed | |
785 if no_proxy == '*': | |
786 bypass_hosts = AllHosts | |
787 | |
788 pi.bypass_hosts = bypass_hosts | |
789 return pi | |
790 | |
791 @classmethod | |
792 def from_url(cls, url, method='http'): | |
793 """ | |
794 Construct a ProxyInfo from a URL (such as http_proxy env var) | |
795 """ | |
796 url = urlparse.urlparse(url) | |
797 username = None | |
798 password = None | |
799 port = None | |
800 if '@' in url[1]: | |
801 ident, host_port = url[1].split('@', 1) | |
802 if ':' in ident: | |
803 username, password = ident.split(':', 1) | |
804 else: | |
805 password = ident | |
806 else: | |
807 host_port = url[1] | |
808 if ':' in host_port: | |
809 host, port = host_port.split(':', 1) | |
810 else: | |
811 host = host_port | |
812 | |
813 if port: | |
814 port = int(port) | |
815 else: | |
816 port = dict(https=443, http=80)[method] | |
817 | |
818 proxy_type = 3 # socks.PROXY_TYPE_HTTP | |
819 return cls( | |
820 proxy_type = proxy_type, | |
821 proxy_host = host, | |
822 proxy_port = port, | |
823 proxy_user = username or None, | |
824 proxy_pass = password or None, | |
825 ) | |
826 | |
827 def applies_to(self, hostname): | 766 def applies_to(self, hostname): |
828 return not self.bypass_host(hostname) | 767 return not self.bypass_host(hostname) |
829 | 768 |
830 def bypass_host(self, hostname): | 769 def bypass_host(self, hostname): |
831 """Has this host been excluded from the proxy config""" | 770 """Has this host been excluded from the proxy config""" |
832 if self.bypass_hosts is AllHosts: | 771 if self.bypass_hosts is AllHosts: |
833 return True | 772 return True |
834 | 773 |
835 bypass = False | 774 bypass = False |
836 for domain in self.bypass_hosts: | 775 for domain in self.bypass_hosts: |
837 if hostname.endswith(domain): | 776 if hostname.endswith(domain): |
838 bypass = True | 777 bypass = True |
839 | 778 |
840 return bypass | 779 return bypass |
841 | 780 |
842 | 781 |
782 def proxy_info_from_environment(method='http'): | |
783 """ | |
784 Read proxy info from the environment variables. | |
785 """ | |
786 if method not in ['http', 'https']: | |
787 return | |
788 | |
789 env_var = method + '_proxy' | |
790 url = os.environ.get(env_var, os.environ.get(env_var.upper())) | |
791 if not url: | |
792 return | |
793 pi = proxy_info_from_url(url, method) | |
794 | |
795 no_proxy = os.environ.get('no_proxy', os.environ.get('NO_PROXY', '')) | |
796 bypass_hosts = [] | |
797 if no_proxy: | |
798 bypass_hosts = no_proxy.split(',') | |
799 # special case, no_proxy=* means all hosts bypassed | |
800 if no_proxy == '*': | |
801 bypass_hosts = AllHosts | |
802 | |
803 pi.bypass_hosts = bypass_hosts | |
804 return pi | |
805 | |
806 def proxy_info_from_url(url, method='http'): | |
807 """ | |
808 Construct a ProxyInfo from a URL (such as http_proxy env var) | |
809 """ | |
810 url = urlparse.urlparse(url) | |
811 username = None | |
812 password = None | |
813 port = None | |
814 if '@' in url[1]: | |
815 ident, host_port = url[1].split('@', 1) | |
816 if ':' in ident: | |
817 username, password = ident.split(':', 1) | |
818 else: | |
819 password = ident | |
820 else: | |
821 host_port = url[1] | |
822 if ':' in host_port: | |
823 host, port = host_port.split(':', 1) | |
824 else: | |
825 host = host_port | |
826 | |
827 if port: | |
828 port = int(port) | |
829 else: | |
830 port = dict(https=443, http=80)[method] | |
831 | |
832 proxy_type = 3 # socks.PROXY_TYPE_HTTP | |
833 return ProxyInfo( | |
834 proxy_type = proxy_type, | |
835 proxy_host = host, | |
836 proxy_port = port, | |
837 proxy_user = username or None, | |
838 proxy_pass = password or None, | |
839 ) | |
840 | |
841 | |
843 class HTTPConnectionWithTimeout(httplib.HTTPConnection): | 842 class HTTPConnectionWithTimeout(httplib.HTTPConnection): |
844 """ | 843 """ |
845 HTTPConnection subclass that supports timeouts | 844 HTTPConnection subclass that supports timeouts |
846 | 845 |
847 All timeouts are in seconds. If None is passed for timeout then | 846 All timeouts are in seconds. If None is passed for timeout then |
848 Python's default timeout for sockets will be used. See for example | 847 Python's default timeout for sockets will be used. See for example |
849 the docs of socket.setdefaulttimeout(): | 848 the docs of socket.setdefaulttimeout(): |
850 http://docs.python.org/library/socket.html#socket.setdefaulttimeout | 849 http://docs.python.org/library/socket.html#socket.setdefaulttimeout |
851 """ | 850 """ |
852 | 851 |
(...skipping 210 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1063 if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: | 1062 if apiproxy_stub_map.apiproxy.GetStub('urlfetch') is None: |
1064 raise ImportError # Bail out; we're not actually running on App Engine. | 1063 raise ImportError # Bail out; we're not actually running on App Engine. |
1065 from google.appengine.api.urlfetch import fetch | 1064 from google.appengine.api.urlfetch import fetch |
1066 from google.appengine.api.urlfetch import InvalidURLError | 1065 from google.appengine.api.urlfetch import InvalidURLError |
1067 from google.appengine.api.urlfetch import DownloadError | 1066 from google.appengine.api.urlfetch import DownloadError |
1068 from google.appengine.api.urlfetch import ResponseTooLargeError | 1067 from google.appengine.api.urlfetch import ResponseTooLargeError |
1069 from google.appengine.api.urlfetch import SSLCertificateError | 1068 from google.appengine.api.urlfetch import SSLCertificateError |
1070 | 1069 |
1071 | 1070 |
1072 class ResponseDict(dict): | 1071 class ResponseDict(dict): |
1073 """Is a dictionary that also has a read() method, so | 1072 """Dictionary with a read() method; can pass off as httplib.HTTPResponse.""" |
1074 that it can pass itself off as an httlib.HTTPResponse().""" | 1073 def __init__(self): |
1074 self.content = kwargs.pop('content', None) | |
1075 return super(ResponseDict, self).__init__(*args, **kwargs) | |
1076 | |
1075 def read(self): | 1077 def read(self): |
1076 pass | 1078 return self.content |
1077 | 1079 |
1078 | 1080 |
1079 class AppEngineHttpConnection(object): | 1081 class AppEngineHttpConnection(object): |
1080 """Emulates an httplib.HTTPConnection object, but actually uses the Google | 1082 """Emulates an httplib.HTTPConnection object, but actually uses the Google |
1081 App Engine urlfetch library. This allows the timeout to be properly used on | 1083 App Engine urlfetch library. This allows the timeout to be properly used on |
1082 Google App Engine, and avoids using httplib, which on Google App Engine is | 1084 Google App Engine, and avoids using httplib, which on Google App Engine is |
1083 just another wrapper around urlfetch. | 1085 just another wrapper around urlfetch. |
1084 """ | 1086 """ |
1085 def __init__(self, host, port=None, key_file=None, cert_file=None, | 1087 def __init__(self, host, port=None, key_file=None, cert_file=None, |
1086 strict=None, timeout=None, proxy_info=None, ca_certs=None, | 1088 strict=None, timeout=None, proxy_info=None, ca_certs=None, |
(...skipping 12 matching lines...) Expand all Loading... | |
1099 # Calculate the absolute URI, which fetch requires | 1101 # Calculate the absolute URI, which fetch requires |
1100 netloc = self.host | 1102 netloc = self.host |
1101 if self.port: | 1103 if self.port: |
1102 netloc = '%s:%s' % (self.host, self.port) | 1104 netloc = '%s:%s' % (self.host, self.port) |
1103 absolute_uri = '%s://%s%s' % (self.scheme, netloc, url) | 1105 absolute_uri = '%s://%s%s' % (self.scheme, netloc, url) |
1104 try: | 1106 try: |
1105 response = fetch(absolute_uri, payload=body, method=method, | 1107 response = fetch(absolute_uri, payload=body, method=method, |
1106 headers=headers, allow_truncated=False, follow_redirects=False, | 1108 headers=headers, allow_truncated=False, follow_redirects=False, |
1107 deadline=self.timeout, | 1109 deadline=self.timeout, |
1108 validate_certificate=self.validate_certificate) | 1110 validate_certificate=self.validate_certificate) |
1109 self.response = ResponseDict(response.headers) | 1111 self.response = ResponseDict(response.headers, response.content) |
1110 self.response['status'] = str(response.status_code) | 1112 self.response['status'] = str(response.status_code) |
1111 self.response['reason'] = httplib.responses.get(response.status_code, 'O k') | 1113 self.response['reason'] = httplib.responses.get(response.status_code, 'O k') |
1112 self.response.status = response.status_code | 1114 self.response.status = response.status_code |
1113 setattr(self.response, 'read', lambda : response.content) | |
1114 | 1115 |
1115 # Make sure the exceptions raised match the exceptions expected. | 1116 # Make sure the exceptions raised match the exceptions expected. |
1116 except InvalidURLError: | 1117 except InvalidURLError: |
1117 raise socket.gaierror('') | 1118 raise socket.gaierror('') |
1118 except (DownloadError, ResponseTooLargeError, SSLCertificateError): | 1119 except (DownloadError, ResponseTooLargeError, SSLCertificateError): |
1119 raise httplib.HTTPException() | 1120 raise httplib.HTTPException() |
1120 | 1121 |
1121 def getresponse(self): | 1122 def getresponse(self): |
1122 if self.response: | 1123 if self.response: |
1123 return self.response | 1124 return self.response |
(...skipping 36 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1160 - ETags | 1161 - ETags |
1161 - compression, | 1162 - compression, |
1162 - HTTPS | 1163 - HTTPS |
1163 - Basic | 1164 - Basic |
1164 - Digest | 1165 - Digest |
1165 - WSSE | 1166 - WSSE |
1166 | 1167 |
1167 and more. | 1168 and more. |
1168 """ | 1169 """ |
1169 def __init__(self, cache=None, timeout=None, | 1170 def __init__(self, cache=None, timeout=None, |
1170 proxy_info=ProxyInfo.from_environment, | 1171 proxy_info=proxy_info_from_environment, |
1171 ca_certs=None, disable_ssl_certificate_validation=False): | 1172 ca_certs=None, disable_ssl_certificate_validation=False): |
1172 """If 'cache' is a string then it is used as a directory name for | 1173 """If 'cache' is a string then it is used as a directory name for |
1173 a disk cache. Otherwise it must be an object that supports the | 1174 a disk cache. Otherwise it must be an object that supports the |
1174 same interface as FileCache. | 1175 same interface as FileCache. |
1175 | 1176 |
1176 All timeouts are in seconds. If None is passed for timeout | 1177 All timeouts are in seconds. If None is passed for timeout |
1177 then Python's default timeout for sockets will be used. See | 1178 then Python's default timeout for sockets will be used. See |
1178 for example the docs of socket.setdefaulttimeout(): | 1179 for example the docs of socket.setdefaulttimeout(): |
1179 http://docs.python.org/library/socket.html#socket.setdefaulttimeout | 1180 http://docs.python.org/library/socket.html#socket.setdefaulttimeout |
1180 | 1181 |
1181 `proxy_info` may be: | 1182 `proxy_info` may be: |
1182 - a callable that takes the http scheme ('http' or 'https') and | 1183 - a callable that takes the http scheme ('http' or 'https') and |
1183 returns a ProxyInfo instance per request. By default, uses | 1184 returns a ProxyInfo instance per request. By default, uses |
1184 ProxyInfo.from_environment. | 1185 proxy_nfo_from_environment. |
1185 - a ProxyInfo instance (static proxy config). | 1186 - a ProxyInfo instance (static proxy config). |
1186 - None (proxy disabled). | 1187 - None (proxy disabled). |
1187 | 1188 |
1188 ca_certs is the path of a file containing root CA certificates for SSL | 1189 ca_certs is the path of a file containing root CA certificates for SSL |
1189 server certificate validation. By default, a CA cert file bundled with | 1190 server certificate validation. By default, a CA cert file bundled with |
1190 httplib2 is used. | 1191 httplib2 is used. |
1191 | 1192 |
1192 If disable_ssl_certificate_validation is true, SSL cert validation will | 1193 If disable_ssl_certificate_validation is true, SSL cert validation will |
1193 not be performed. | 1194 not be performed. |
1194 """ | 1195 """ |
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1228 | 1229 |
1229 self.ignore_etag = False | 1230 self.ignore_etag = False |
1230 | 1231 |
1231 self.force_exception_to_status_code = False | 1232 self.force_exception_to_status_code = False |
1232 | 1233 |
1233 self.timeout = timeout | 1234 self.timeout = timeout |
1234 | 1235 |
1235 # Keep Authorization: headers on a redirect. | 1236 # Keep Authorization: headers on a redirect. |
1236 self.forward_authorization_headers = False | 1237 self.forward_authorization_headers = False |
1237 | 1238 |
1239 def __getstate__(self): | |
1240 state_dict = copy.copy(self.__dict__) | |
1241 # In case request is augmented by some foreign object such as | |
1242 # credentials which handle auth | |
jcgregorio_google
2012/09/05 15:20:07
So are you planning on reinstating the credentials
dhermes
2012/09/05 15:23:15
Yes. Given that the request attribute could be wra
| |
1243 if 'request' in state_dict: | |
1244 del state_dict['request'] | |
1245 return state_dict | |
1246 | |
1247 def __setstate__(self, state): | |
1248 self.__dict__.update(state) | |
1249 | |
1238 def _auth_from_challenge(self, host, request_uri, headers, response, content ): | 1250 def _auth_from_challenge(self, host, request_uri, headers, response, content ): |
1239 """A generator that creates Authorization objects | 1251 """A generator that creates Authorization objects |
1240 that can be applied to requests. | 1252 that can be applied to requests. |
1241 """ | 1253 """ |
1242 challenges = _parse_www_authenticate(response, 'www-authenticate') | 1254 challenges = _parse_www_authenticate(response, 'www-authenticate') |
1243 for cred in self.credentials.iter(host): | 1255 for cred in self.credentials.iter(host): |
1244 for scheme in AUTH_SCHEME_ORDER: | 1256 for scheme in AUTH_SCHEME_ORDER: |
1245 if challenges.has_key(scheme): | 1257 if challenges.has_key(scheme): |
1246 yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, h eaders, response, content, self) | 1258 yield AUTH_SCHEME_CLASSES[scheme](cred, host, request_uri, h eaders, response, content, self) |
1247 | 1259 |
(...skipping 414 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
1662 self[key.lower()] = value | 1674 self[key.lower()] = value |
1663 self.status = int(self.get('status', self.status)) | 1675 self.status = int(self.get('status', self.status)) |
1664 self.reason = self.get('reason', self.reason) | 1676 self.reason = self.get('reason', self.reason) |
1665 | 1677 |
1666 | 1678 |
1667 def __getattr__(self, name): | 1679 def __getattr__(self, name): |
1668 if name == 'dict': | 1680 if name == 'dict': |
1669 return self | 1681 return self |
1670 else: | 1682 else: |
1671 raise AttributeError, name | 1683 raise AttributeError, name |
OLD | NEW |