diff options
Diffstat (limited to 'requests/models.py')
| -rw-r--r-- | requests/models.py | 1052 |
1 files changed, 499 insertions, 553 deletions
diff --git a/requests/models.py b/requests/models.py index 84a2ec6..8fd9735 100644 --- a/requests/models.py +++ b/requests/models.py @@ -7,558 +7,488 @@ requests.models This module contains the primary objects that power Requests. """ -import os -import urllib +import collections +import logging +import datetime -from urlparse import urlparse, urlunparse, urljoin, urlsplit -from datetime import datetime - -from .hooks import dispatch_hook, HOOKS +from io import BytesIO, UnsupportedOperation +from .hooks import default_hooks from .structures import CaseInsensitiveDict -from .status_codes import codes -from .packages import oreos -from .auth import HTTPBasicAuth, HTTPProxyAuth -from .packages.urllib3.response import HTTPResponse -from .packages.urllib3.exceptions import MaxRetryError -from .packages.urllib3.exceptions import SSLError as _SSLError -from .packages.urllib3.exceptions import HTTPError as _HTTPError -from .packages.urllib3 import connectionpool, poolmanager + +from .auth import HTTPBasicAuth +from .cookies import cookiejar_from_dict, get_cookie_header from .packages.urllib3.filepost import encode_multipart_formdata +from .packages.urllib3.util import parse_url from .exceptions import ( - ConnectionError, HTTPError, RequestException, Timeout, TooManyRedirects, - URLRequired, SSLError) + HTTPError, RequestException, MissingSchema, InvalidURL, + ChunkedEncodingError) from .utils import ( - get_encoding_from_headers, stream_decode_response_unicode, - stream_decompress, guess_filename, requote_path) - -# Import chardet if it is available. -try: - import chardet -except ImportError: - pass - -REDIRECT_STATI = (codes.moved, codes.found, codes.other, codes.temporary_moved) - - - -class Request(object): - """The :class:`Request <Request>` object. It carries out all functionality of - Requests. Recommended interface is with the Requests functions. - """ - - def __init__(self, - url=None, - headers=dict(), - files=None, - method=None, - data=dict(), - params=dict(), - auth=None, - cookies=None, - timeout=None, - redirect=False, - allow_redirects=False, - proxies=None, - hooks=None, - config=None, - _poolmanager=None, - verify=None, - session=None): - - #: Float describes the timeout of the request. - # (Use socket.setdefaulttimeout() as fallback) - self.timeout = timeout - - #: Request URL. - self.url = url - - #: Dictionary of HTTP Headers to attach to the :class:`Request <Request>`. - self.headers = dict(headers or []) - - #: Dictionary of files to multipart upload (``{filename: content}``). - self.files = files - - #: HTTP Method to use. - self.method = method - - #: Dictionary or byte of request body data to attach to the - #: :class:`Request <Request>`. - self.data = None - - #: Dictionary or byte of querystring data to attach to the - #: :class:`Request <Request>`. - self.params = None - - #: True if :class:`Request <Request>` is part of a redirect chain (disables history - #: and HTTPError storage). - self.redirect = redirect + guess_filename, get_auth_from_url, requote_uri, + stream_decode_response_unicode, to_key_val_list, parse_header_links, + iter_slices, guess_json_utf, super_len, to_native_string) +from .compat import ( + cookielib, urlunparse, urlsplit, urlencode, str, bytes, StringIO, + is_py2, chardet, json, builtin_str, basestring, IncompleteRead) - #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) - self.allow_redirects = allow_redirects +CONTENT_CHUNK_SIZE = 10 * 1024 +ITER_CHUNK_SIZE = 512 - # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) - self.proxies = dict(proxies or []) +log = logging.getLogger(__name__) - self.data, self._enc_data = self._encode_params(data) - self.params, self._enc_params = self._encode_params(params) - #: :class:`Response <Response>` instance, containing - #: content and metadata of HTTP Response, once :attr:`sent <send>`. - self.response = Response() - - #: Authentication tuple or object to attach to :class:`Request <Request>`. - self.auth = auth - - #: CookieJar to attach to :class:`Request <Request>`. - self.cookies = dict(cookies or []) +class RequestEncodingMixin(object): + @property + def path_url(self): + """Build the path URL to use.""" - #: Dictionary of configurations for this request. - self.config = dict(config or []) + url = [] - #: True if Request has been sent. - self.sent = False + p = urlsplit(self.url) - #: Event-handling hooks. - self.hooks = {} + path = p.path + if not path: + path = '/' - for event in HOOKS: - self.hooks[event] = [] + url.append(path) - hooks = hooks or {} + query = p.query + if query: + url.append('?') + url.append(query) - for (k, v) in hooks.items(): - self.register_hook(event=k, hook=v) + return ''.join(url) - #: Session. - self.session = session + @staticmethod + def _encode_params(data): + """Encode parameters in a piece of data. - #: SSL Verification. - self.verify = verify + Will successfully encode parameters when passed as a dict or a list of + 2-tuples. Order is retained if data is a list of 2-tuples but arbitrary + if parameters are supplied as a dict. + """ - if headers: - headers = CaseInsensitiveDict(self.headers) + if isinstance(data, (str, bytes)): + return data + elif hasattr(data, 'read'): + return data + elif hasattr(data, '__iter__'): + result = [] + for k, vs in to_key_val_list(data): + if isinstance(vs, basestring) or not hasattr(vs, '__iter__'): + vs = [vs] + for v in vs: + if v is not None: + result.append( + (k.encode('utf-8') if isinstance(k, str) else k, + v.encode('utf-8') if isinstance(v, str) else v)) + return urlencode(result, doseq=True) else: - headers = CaseInsensitiveDict() + return data - # Add configured base headers. - for (k, v) in self.config.get('base_headers', {}).items(): - if k not in headers: - headers[k] = v + @staticmethod + def _encode_files(files, data): + """Build the body for a multipart/form-data request. - self.headers = headers - self._poolmanager = _poolmanager + Will successfully encode files when passed as a dict or a list of + 2-tuples. Order is retained if data is a list of 2-tuples but abritrary + if parameters are supplied as a dict. - # Pre-request hook. - r = dispatch_hook('pre_request', hooks, self) - self.__dict__.update(r.__dict__) + """ + if (not files): + raise ValueError("Files must be provided.") + elif isinstance(data, basestring): + raise ValueError("Data must not be a string.") + new_fields = [] + fields = to_key_val_list(data or {}) + files = to_key_val_list(files or {}) - def __repr__(self): - return '<Request [%s]>' % (self.method) + for field, val in fields: + if isinstance(val, basestring) or not hasattr(val, '__iter__'): + val = [val] + for v in val: + if v is not None: + # Don't call str() on bytestrings: in Py3 it all goes wrong. + if not isinstance(v, bytes): + v = str(v) + new_fields.append( + (field.decode('utf-8') if isinstance(field, bytes) else field, + v.encode('utf-8') if isinstance(v, str) else v)) - def _build_response(self, resp): - """Build internal :class:`Response <Response>` object - from given response. - """ + for (k, v) in files: + # support for explicit filename + ft = None + if isinstance(v, (tuple, list)): + if len(v) == 2: + fn, fp = v + else: + fn, fp, ft = v + else: + fn = guess_filename(v) or k + fp = v + if isinstance(fp, str): + fp = StringIO(fp) + if isinstance(fp, bytes): + fp = BytesIO(fp) - def build(resp): + if ft: + new_v = (fn, fp.read(), ft) + else: + new_v = (fn, fp.read()) + new_fields.append((k, new_v)) - response = Response() + body, content_type = encode_multipart_formdata(new_fields) - # Pass settings over. - response.config = self.config + return body, content_type - if resp: - # Fallback to None if there's no status_code, for whatever reason. - response.status_code = getattr(resp, 'status', None) +class RequestHooksMixin(object): + def register_hook(self, event, hook): + """Properly register a hook.""" - # Make headers case-insensitive. - response.headers = CaseInsensitiveDict(getattr(resp, 'headers', None)) + if event not in self.hooks: + raise ValueError('Unsupported event specified, with event name "%s"' % (event)) - # Set encoding. - response.encoding = get_encoding_from_headers(response.headers) + if isinstance(hook, collections.Callable): + self.hooks[event].append(hook) + elif hasattr(hook, '__iter__'): + self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) - # Start off with our local cookies. - cookies = self.cookies or dict() + def deregister_hook(self, event, hook): + """Deregister a previously registered hook. + Returns True if the hook existed, False if not. + """ - # Add new cookies from the server. - if 'set-cookie' in response.headers: - cookie_header = response.headers['set-cookie'] - cookies = oreos.dict_from_string(cookie_header) + try: + self.hooks[event].remove(hook) + return True + except ValueError: + return False - # Save cookies in Response. - response.cookies = cookies - # No exceptions were harmed in the making of this request. - response.error = getattr(resp, 'error', None) +class Request(RequestHooksMixin): + """A user-created :class:`Request <Request>` object. - # Save original response for later. - response.raw = resp - response.url = self.full_url.decode('utf-8') + Used to prepare a :class:`PreparedRequest <PreparedRequest>`, which is sent to the server. - return response + :param method: HTTP method to use. + :param url: URL to send. + :param headers: dictionary of headers to send. + :param files: dictionary of {filename: fileobject} files to multipart upload. + :param data: the body to attach the request. If a dictionary is provided, form-encoding will take place. + :param params: dictionary of URL parameters to append to the URL. + :param auth: Auth handler or (user, pass) tuple. + :param cookies: dictionary or CookieJar of cookies to attach to this request. + :param hooks: dictionary of callback hooks, for internal usage. - history = [] + Usage:: - r = build(resp) - cookies = self.cookies - self.cookies.update(r.cookies) + >>> import requests + >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req.prepare() + <PreparedRequest [GET]> - if r.status_code in REDIRECT_STATI and not self.redirect: + """ + def __init__(self, + method=None, + url=None, + headers=None, + files=None, + data=None, + params=None, + auth=None, + cookies=None, + hooks=None): - while ( - ('location' in r.headers) and - ((r.status_code is codes.see_other) or (self.allow_redirects)) - ): + # Default empty dicts for dict params. + data = [] if data is None else data + files = [] if files is None else files + headers = {} if headers is None else headers + params = {} if params is None else params + hooks = {} if hooks is None else hooks - if not len(history) < self.config.get('max_redirects'): - raise TooManyRedirects() + self.hooks = default_hooks() + for (k, v) in list(hooks.items()): + self.register_hook(event=k, hook=v) - history.append(r) + self.method = method + self.url = url + self.headers = headers + self.files = files + self.data = data + self.params = params + self.auth = auth + self.cookies = cookies - url = r.headers['location'] + def __repr__(self): + return '<Request [%s]>' % (self.method) - # Handle redirection without scheme (see: RFC 1808 Section 4) - if url.startswith('//'): - parsed_rurl = urlparse(r.url) - url = '%s:%s' % (parsed_rurl.scheme, url) + def prepare(self): + """Constructs a :class:`PreparedRequest <PreparedRequest>` for transmission and returns it.""" + p = PreparedRequest() + p.prepare( + method=self.method, + url=self.url, + headers=self.headers, + files=self.files, + data=self.data, + params=self.params, + auth=self.auth, + cookies=self.cookies, + hooks=self.hooks, + ) + return p - # Facilitate non-RFC2616-compliant 'location' headers - # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') - if not urlparse(url).netloc: - url = urljoin(r.url, url) - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 - if r.status_code is codes.see_other: - method = 'GET' - else: - method = self.method +class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): + """The fully mutable :class:`PreparedRequest <PreparedRequest>` object, + containing the exact bytes that will be sent to the server. - # Remove the cookie headers that were sent. - headers = self.headers - try: - del headers['Cookie'] - except KeyError: - pass + Generated from either a :class:`Request <Request>` object or manually. - request = Request( - url=url, - headers=headers, - files=self.files, - method=method, - params=self.session.params, - auth=self.auth, - cookies=cookies, - redirect=True, - config=self.config, - timeout=self.timeout, - _poolmanager=self._poolmanager, - proxies = self.proxies, - verify = self.verify, - session = self.session - ) + Usage:: - request.send() - cookies.update(request.response.cookies) - r = request.response - self.cookies.update(r.cookies) + >>> import requests + >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> r = req.prepare() + <PreparedRequest [GET]> - r.history = history + >>> s = requests.Session() + >>> s.send(r) + <Response [200]> - self.response = r - self.response.request = self - self.response.cookies.update(self.cookies) + """ + def __init__(self): + #: HTTP verb to send to the server. + self.method = None + #: HTTP URL to send the request to. + self.url = None + #: dictionary of HTTP headers. + self.headers = None + #: request body to send to the server. + self.body = None + #: dictionary of callback hooks, for internal usage. + self.hooks = default_hooks() - @staticmethod - def _encode_params(data): - """Encode parameters in a piece of data. + def prepare(self, method=None, url=None, headers=None, files=None, + data=None, params=None, auth=None, cookies=None, hooks=None): + """Prepares the the entire request with the given parameters.""" - If the data supplied is a dictionary, encodes each parameter in it, and - returns a list of tuples containing the encoded parameters, and a urlencoded - version of that. + self.prepare_method(method) + self.prepare_url(url, params) + self.prepare_headers(headers) + self.prepare_cookies(cookies) + self.prepare_body(data, files) + self.prepare_auth(auth, url) + # Note that prepare_auth must be last to enable authentication schemes + # such as OAuth to work on a fully prepared request. - Otherwise, assumes the data is already encoded appropriately, and - returns it twice. - """ + # This MUST go after prepare_auth. Authenticators could add a hook + self.prepare_hooks(hooks) - if hasattr(data, '__iter__'): - data = dict(data) + def __repr__(self): + return '<PreparedRequest [%s]>' % (self.method) - if hasattr(data, 'items'): - result = [] - for k, vs in data.items(): - for v in isinstance(vs, list) and vs or [vs]: - result.append((k.encode('utf-8') if isinstance(k, unicode) else k, - v.encode('utf-8') if isinstance(v, unicode) else v)) - return result, urllib.urlencode(result, doseq=True) - else: - return data, data + def copy(self): + p = PreparedRequest() + p.method = self.method + p.url = self.url + p.headers = self.headers + p.body = self.body + p.hooks = self.hooks + return p - @property - def full_url(self): - """Build the actual URL to use.""" + def prepare_method(self, method): + """Prepares the given HTTP method.""" + self.method = method + if self.method is not None: + self.method = self.method.upper() - if not self.url: - raise URLRequired() + def prepare_url(self, url, params): + """Prepares the given HTTP URL.""" + #: Accept objects that have string representations. + try: + url = unicode(url) + except NameError: + # We're on Python 3. + url = str(url) + except UnicodeDecodeError: + pass # Support for unicode domain names and paths. - scheme, netloc, path, params, query, fragment = urlparse(self.url) + scheme, auth, host, port, path, query, fragment = parse_url(url) if not scheme: - raise ValueError("Invalid URL %r: No schema supplied" %self.url) - - netloc = netloc.encode('idna') - - if isinstance(path, unicode): - path = path.encode('utf-8') - - path = requote_path(path) - - url = str(urlunparse([ scheme, netloc, path, params, query, fragment ])) - - if self._enc_params: - if urlparse(url).query: - return '%s&%s' % (url, self._enc_params) - else: - return '%s?%s' % (url, self._enc_params) - else: - return url - - @property - def path_url(self): - """Build the path URL to use.""" + raise MissingSchema("Invalid URL %r: No schema supplied" % url) - url = [] + if not host: + raise InvalidURL("Invalid URL %r: No host supplied" % url) - p = urlsplit(self.full_url) + # Only want to apply IDNA to the hostname + try: + host = host.encode('idna').decode('utf-8') + except UnicodeError: + raise InvalidURL('URL has an invalid label.') - # Proxies use full URLs. - if p.scheme in self.proxies: - return self.full_url + # Carefully reconstruct the network location + netloc = auth or '' + if netloc: + netloc += '@' + netloc += host + if port: + netloc += ':' + str(port) - path = p.path + # Bare domains aren't valid URLs. if not path: path = '/' - url.append(path) - - query = p.query - if query: - url.append('?') - url.append(query) - - return ''.join(url) - - def register_hook(self, event, hook): - """Properly register a hook.""" - - return self.hooks[event].append(hook) + if is_py2: + if isinstance(scheme, str): + scheme = scheme.encode('utf-8') + if isinstance(netloc, str): + netloc = netloc.encode('utf-8') + if isinstance(path, str): + path = path.encode('utf-8') + if isinstance(query, str): + query = query.encode('utf-8') + if isinstance(fragment, str): + fragment = fragment.encode('utf-8') + enc_params = self._encode_params(params) + if enc_params: + if query: + query = '%s&%s' % (query, enc_params) + else: + query = enc_params - def send(self, anyway=False, prefetch=False): - """Sends the request. Returns True of successful, false if not. - If there was an HTTPError during transmission, - self.response.status_code will contain the HTTPError code. + url = requote_uri(urlunparse([scheme, netloc, path, None, query, fragment])) + self.url = url - Once a request is successfully sent, `sent` will equal True. + def prepare_headers(self, headers): + """Prepares the given HTTP headers.""" - :param anyway: If True, request will be sent, even if it has - already been sent. - """ + if headers: + self.headers = CaseInsensitiveDict((to_native_string(name), value) for name, value in headers.items()) + else: + self.headers = CaseInsensitiveDict() - # Build the URL - url = self.full_url + def prepare_body(self, data, files): + """Prepares the given HTTP body data.""" - # Logging - if self.config.get('verbose'): - self.config.get('verbose').write('%s %s %s\n' % ( - datetime.now().isoformat(), self.method, url - )) + # Check if file, fo, generator, iterator. + # If not, run through normal process. # Nottin' on you. body = None content_type = None + length = None - # Multi-part file uploads. - if self.files: - if not isinstance(self.data, basestring): + is_stream = all([ + hasattr(data, '__iter__'), + not isinstance(data, basestring), + not isinstance(data, list), + not isinstance(data, dict) + ]) - try: - fields = self.data.copy() - except AttributeError: - fields = dict(self.data) - - for (k, v) in self.files.items(): - # support for explicit filename - if isinstance(v, (tuple, list)): - fn, fp = v - else: - fn = guess_filename(v) or k - fp = v - fields.update({k: (fn, fp.read())}) - - (body, content_type) = encode_multipart_formdata(fields) - else: - pass - # TODO: Conflict? - else: - if self.data: - - body = self._enc_data - if isinstance(self.data, basestring): - content_type = None - else: - content_type = 'application/x-www-form-urlencoded' - - # Add content-type if it wasn't explicitly provided. - if (content_type) and (not 'content-type' in self.headers): - self.headers['Content-Type'] = content_type - - if self.auth: - if isinstance(self.auth, tuple) and len(self.auth) == 2: - # special-case basic HTTP auth - self.auth = HTTPBasicAuth(*self.auth) - - # Allow auth to make its changes. - r = self.auth(self) + try: + length = super_len(data) + except (TypeError, AttributeError, UnsupportedOperation): + length = None - # Update self to reflect the auth changes. - self.__dict__.update(r.__dict__) + if is_stream: + body = data - _p = urlparse(url) - proxy = self.proxies.get(_p.scheme) + if files: + raise NotImplementedError('Streamed bodies and files are mutually exclusive.') - if proxy: - conn = poolmanager.proxy_from_url(proxy) - _proxy = urlparse(proxy) - if '@' in _proxy.netloc: - auth, url = _proxy.netloc.split('@', 1) - self.proxy_auth = HTTPProxyAuth(*auth.split(':', 1)) - r = self.proxy_auth(self) - self.__dict__.update(r.__dict__) - else: - # Check to see if keep_alive is allowed. - if self.config.get('keep_alive'): - conn = self._poolmanager.connection_from_url(url) + if length is not None: + self.headers['Content-Length'] = str(length) else: - conn = connectionpool.connection_from_url(url) - - if url.startswith('https') and self.verify: - - cert_loc = None - - # Allow self-specified cert location. - if self.verify is not True: - cert_loc = self.verify - - - # Look for configuration. - if not cert_loc: - cert_loc = os.environ.get('REQUESTS_CA_BUNDLE') - - # Curl compatiblity. - if not cert_loc: - cert_loc = os.environ.get('CURL_CA_BUNDLE') - - # Use the awesome certifi list. - if not cert_loc: - cert_loc = __import__('certifi').where() - - conn.cert_reqs = 'CERT_REQUIRED' - conn.ca_certs = cert_loc + self.headers['Transfer-Encoding'] = 'chunked' else: - conn.cert_reqs = 'CERT_NONE' - conn.ca_certs = None - - if not self.sent or anyway: - - if self.cookies: - - # Skip if 'cookie' header is explicitly set. - if 'cookie' not in self.headers: + # Multi-part file uploads. + if files: + (body, content_type) = self._encode_files(files, data) + else: + if data: + body = self._encode_params(data) + if isinstance(data, str) or isinstance(data, builtin_str) or hasattr(data, 'read'): + content_type = None + else: + content_type = 'application/x-www-form-urlencoded' - # Simple cookie with our dict. - c = oreos.monkeys.SimpleCookie() - for (k, v) in self.cookies.items(): - c[k] = v + self.prepare_content_length(body) - # Turn it into a header. - cookie_header = c.output(header='', sep='; ').strip() + # Add content-type if it wasn't explicitly provided. + if (content_type) and (not 'content-type' in self.headers): + self.headers['Content-Type'] = content_type - # Attach Cookie header to request. - self.headers['Cookie'] = cookie_header + self.body = body - try: - # The inner try .. except re-raises certain exceptions as - # internal exception types; the outer suppresses exceptions - # when safe mode is set. - try: - # Send the request. - r = conn.urlopen( - method=self.method, - url=self.path_url, - body=body, - headers=self.headers, - redirect=False, - assert_same_host=False, - preload_content=False, - decode_content=True, - retries=self.config.get('max_retries', 0), - timeout=self.timeout, - ) - self.sent = True + def prepare_content_length(self, body): + if hasattr(body, 'seek') and hasattr(body, 'tell'): + body.seek(0, 2) + self.headers['Content-Length'] = str(body.tell()) + body.seek(0, 0) + elif body is not None: + l = super_len(body) + if l: + self.headers['Content-Length'] = str(l) + elif self.method not in ('GET', 'HEAD'): + self.headers['Content-Length'] = '0' - except MaxRetryError, e: - raise ConnectionError(e) + def prepare_auth(self, auth, url=''): + """Prepares the given HTTP auth data.""" - except (_SSLError, _HTTPError), e: - if self.verify and isinstance(e, _SSLError): - raise SSLError(e) + # If no Auth is explicitly provided, extract it from the URL first. + if auth is None: + url_auth = get_auth_from_url(self.url) + auth = url_auth if any(url_auth) else None - raise Timeout('Request timed out.') + if auth: + if isinstance(auth, tuple) and len(auth) == 2: + # special-case basic HTTP auth + auth = HTTPBasicAuth(*auth) - except RequestException, e: - if self.config.get('safe_mode', False): - # In safe mode, catch the exception and attach it to - # a blank urllib3.HTTPResponse object. - r = HTTPResponse() - r.error = e - else: - raise + # Allow auth to make its changes. + r = auth(self) - self._build_response(r) + # Update self to reflect the auth changes. + self.__dict__.update(r.__dict__) - # Response manipulation hook. - self.response = dispatch_hook('response', self.hooks, self.response) + # Recompute Content-Length + self.prepare_content_length(self.body) - # Post-request hook. - r = dispatch_hook('post_request', self.hooks, self) - self.__dict__.update(r.__dict__) + def prepare_cookies(self, cookies): + """Prepares the given HTTP cookie data.""" - # If prefetch is True, mark content as consumed. - if prefetch: - # Save the response. - self.response.content + if isinstance(cookies, cookielib.CookieJar): + cookies = cookies + else: + cookies = cookiejar_from_dict(cookies) - if self.config.get('danger_mode'): - self.response.raise_for_status() + if 'cookie' not in self.headers: + cookie_header = get_cookie_header(cookies, self) + if cookie_header is not None: + self.headers['Cookie'] = cookie_header - return self.sent + def prepare_hooks(self, hooks): + """Prepares the given hooks.""" + for event in hooks: + self.register_hook(event, hooks[event]) class Response(object): - """The core :class:`Response <Response>` object. All - :class:`Request <Request>` objects contain a - :class:`response <Response>` attribute, which is an instance - of this class. + """The :class:`Response <Response>` object, which contains a + server's response to an HTTP request. """ def __init__(self): + super(Response, self).__init__() - self._content = None + self._content = False self._content_consumed = False #: Integer Code of responded HTTP Status. @@ -570,162 +500,146 @@ class Response(object): self.headers = CaseInsensitiveDict() #: File-like object representation of response (for advanced usage). + #: Requires that ``stream=True` on the request. + # This requirement does not apply for use internally to Requests. self.raw = None #: Final URL location of Response. self.url = None - #: Resulting :class:`HTTPError` of request, if one occurred. - self.error = None - - #: Encoding to decode with when accessing r.content. + #: Encoding to decode with when accessing r.text. self.encoding = None #: A list of :class:`Response <Response>` objects from #: the history of the Request. Any redirect responses will end - #: up here. + #: up here. The list is sorted from the oldest to the most recent request. self.history = [] - #: The :class:`Request <Request>` that created the Response. - self.request = None - - #: A dictionary of Cookies the server sent back. - self.cookies = {} + self.reason = None - #: Dictionary of configurations for this request. - self.config = {} + #: A CookieJar of Cookies the server sent back. + self.cookies = cookiejar_from_dict({}) + #: The amount of time elapsed between sending the request + #: and the arrival of the response (as a timedelta) + self.elapsed = datetime.timedelta(0) def __repr__(self): return '<Response [%s]>' % (self.status_code) + def __bool__(self): + """Returns true if :attr:`status_code` is 'OK'.""" + return self.ok + def __nonzero__(self): """Returns true if :attr:`status_code` is 'OK'.""" return self.ok + def __iter__(self): + """Allows you to use a response as an iterator.""" + return self.iter_content(128) + @property def ok(self): try: self.raise_for_status() - except HTTPError: + except RequestException: return False return True + @property + def apparent_encoding(self): + """The apparent encoding, provided by the lovely Charade library + (Thanks, Ian!).""" + return chardet.detect(self.content)['encoding'] - def iter_content(self, chunk_size=10 * 1024, decode_unicode=False): - """Iterates over the response data. This avoids reading the content - at once into memory for large responses. The chunk size is the number - of bytes it should read into memory. This is not necessarily the - length of each item returned as decoding can take place. + def iter_content(self, chunk_size=1, decode_unicode=False): + """Iterates over the response data. When stream=True is set on the + request, this avoids reading the content at once into memory for + large responses. The chunk size is the number of bytes it should + read into memory. This is not necessarily the length of each item + returned as decoding can take place. """ if self._content_consumed: - raise RuntimeError( - 'The content for this response was already consumed' - ) + # simulate reading small chunks of the content + return iter_slices(self._content, chunk_size) def generate(): - while 1: - chunk = self.raw.read(chunk_size) - if not chunk: - break - yield chunk - self._content_consumed = True - - def generate_chunked(): - resp = self.raw._original_response - fp = resp.fp - if resp.chunk_left is not None: - pending_bytes = resp.chunk_left - while pending_bytes: - chunk = fp.read(min(chunk_size, pending_bytes)) - pending_bytes-=len(chunk) - yield chunk - fp.read(2) # throw away crlf - while 1: - #XXX correct line size? (httplib has 64kb, seems insane) - pending_bytes = fp.readline(40).strip() - pending_bytes = int(pending_bytes, 16) - if pending_bytes == 0: - break - while pending_bytes: - chunk = fp.read(min(chunk_size, pending_bytes)) - pending_bytes-=len(chunk) + try: + # Special case for urllib3. + try: + for chunk in self.raw.stream(chunk_size, + decode_content=True): + yield chunk + except IncompleteRead as e: + raise ChunkedEncodingError(e) + except AttributeError: + # Standard file-like object. + while 1: + chunk = self.raw.read(chunk_size) + if not chunk: + break yield chunk - fp.read(2) # throw away crlf - self._content_consumed = True - fp.close() - - if getattr(getattr(self.raw, '_original_response', None), 'chunked', False): - gen = generate_chunked() - else: - gen = generate() + self._content_consumed = True - if 'gzip' in self.headers.get('content-encoding', ''): - gen = stream_decompress(gen, mode='gzip') - elif 'deflate' in self.headers.get('content-encoding', ''): - gen = stream_decompress(gen, mode='deflate') + gen = generate() if decode_unicode: gen = stream_decode_response_unicode(gen, self) return gen - - def iter_lines(self, chunk_size=10 * 1024, decode_unicode=None): - """Iterates over the response data, one line at a time. This - avoids reading the content at once into memory for large - responses. + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None): + """Iterates over the response data, one line at a time. When + stream=True is set on the request, this avoids reading the + content at once into memory for large responses. """ - #TODO: why rstrip by default pending = None - for chunk in self.iter_content(chunk_size, decode_unicode=decode_unicode): + for chunk in self.iter_content(chunk_size=chunk_size, + decode_unicode=decode_unicode): if pending is not None: chunk = pending + chunk - lines = chunk.splitlines(True) - - for line in lines[:-1]: - yield line.rstrip() + lines = chunk.splitlines() - # Save the last part of the chunk for next iteration, to keep full line together - # lines may be empty for the last chunk of a chunked response - - if lines: - pending = lines[-1] - #if pending is a complete line, give it baack - if pending[-1] == '\n': - yield pending.rstrip() - pending = None + if lines and lines[-1] and chunk and lines[-1][-1] == chunk[-1]: + pending = lines.pop() else: pending = None - # Yield the last line - if pending is not None: - yield pending.rstrip() + for line in lines: + yield line + if pending is not None: + yield pending @property def content(self): """Content of the response, in bytes.""" - if self._content is None: + if self._content is False: # Read the contents. try: if self._content_consumed: raise RuntimeError( 'The content for this response was already consumed') - self._content = self.raw.read() + if self.status_code == 0: + self._content = None + else: + self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() + except AttributeError: self._content = None self._content_consumed = True + # don't need to release the connection; that's been handled by urllib3 + # since we exhausted the data. return self._content - @property def text(self): """Content of the response, in unicode. @@ -738,47 +652,79 @@ class Response(object): content = None encoding = self.encoding - # Fallback to auto-detected encoding if chardet is available. - if self.encoding is None: - try: - detected = chardet.detect(self.content) or {} - encoding = detected.get('encoding') + if not self.content: + return str('') - # Trust that chardet isn't available or something went terribly wrong. - except Exception: - pass + # Fallback to auto-detected encoding. + if self.encoding is None: + encoding = self.apparent_encoding # Decode unicode from given encoding. try: - content = unicode(self.content, encoding) - except UnicodeError, TypeError: - pass + content = str(self.content, encoding, errors='replace') + except (LookupError, TypeError): + # A LookupError is raised if the encoding was not found which could + # indicate a misspelling or similar mistake. + # + # A TypeError can be raised if encoding is None + # + # So we try blindly encoding. + content = str(self.content, errors='replace') - # Try to fall back: - if not content: - try: - content = unicode(content, encoding, errors='replace') - except UnicodeError, TypeError: - pass + return content + def json(self, **kwargs): + """Returns the json-encoded content of a response, if any. + :param \*\*kwargs: Optional arguments that ``json.loads`` takes. + """ - return content + if not self.encoding and len(self.content) > 3: + # No encoding set. JSON RFC 4627 section 3 states we should expect + # UTF-8, -16 or -32. Detect which one to use; If the detection or + # decoding fails, fall back to `self.text` (using chardet to make + # a best guess). + encoding = guess_json_utf(self.content) + if encoding is not None: + return json.loads(self.content.decode(encoding), **kwargs) + return json.loads(self.text or self.content, **kwargs) + + @property + def links(self): + """Returns the parsed header links of the response, if any.""" + + header = self.headers.get('link') + + # l = MultiDict() + l = {} + if header: + links = parse_header_links(header) + + for link in links: + key = link.get('rel') or link.get('url') + l[key] = link + + return l def raise_for_status(self): - """Raises stored :class:`HTTPError` or :class:`URLError`, if one occurred.""" + """Raises stored :class:`HTTPError`, if one occurred.""" - if self.error: - raise self.error + http_error_msg = '' - if (self.status_code >= 300) and (self.status_code < 400): - raise HTTPError('%s Redirection' % self.status_code) + if 400 <= self.status_code < 500: + http_error_msg = '%s Client Error: %s' % (self.status_code, self.reason) - elif (self.status_code >= 400) and (self.status_code < 500): - raise HTTPError('%s Client Error' % self.status_code) + elif 500 <= self.status_code < 600: + http_error_msg = '%s Server Error: %s' % (self.status_code, self.reason) - elif (self.status_code >= 500) and (self.status_code < 600): - raise HTTPError('%s Server Error' % self.status_code) + if http_error_msg: + raise HTTPError(http_error_msg, response=self) + def close(self): + """Closes the underlying file descriptor and releases the connection + back to the pool. + *Note: Should not normally need to be called explicitly.* + """ + return self.raw.release_conn() |