Requests v0.2.0 源码阅读

Requests v0.2.0 源码阅读

v0.2.0

git clone https://github.com/requests/requests

从 https://github.com/requests/requests/releases?after=v0.3.0 知道 v0.2.0 发布时的 commit为 https://github.com/requests/requests/commit/d2427ecae751a533ddd9026849dd19cfaa3394f4 。检出。

项目结构

name	usage
docs	保存文档
requests	保存源代码
.gitignore	略
HISTORY.rst	历史
LICENSE	协议
README.rst	readme
setup.py	安装
test_requests.py	测试

test_requests.py

定义如上方法，用于进行功能测试。

requests

主要关注 core.py

UML图：

Structure：

主要实现四种类：请求基类_Request、请求类Request、响应类Response、认证AuthObject，七种方法：get、post、put、delete和认证相关的方法，四种异常类。

_Request 类

对urllib2.Request对象 的封装，允许对请求方法进行s手动设置。

class _Request(urllib2.Request):
    """Hidden wrapper around the urllib2.Request object. Allows for manual
    setting of HTTP methods.
    """

    def __init__(self, url,
                    data=None, headers={}, origin_req_host=None,
                    unverifiable=False, method=None):
        urllib2.Request.__init__( self, url, data, headers, origin_req_host,
                                  unverifiable)
		# 设置请求方法
        self.method = method 
	
	# 获取请求方法
    def get_method(self):
        if self.method:
            return self.method

        return urllib2.Request.get_method(self)

Request 类

附上一些私有变量和私有方法：

class Request(object):
	"""The :class:`Request` object. It carries out all functionality of
	Requests. Recommended interface is with the Requests functions.
	
	"""
    
	_METHODS = ('GET', 'HEAD', 'PUT', 'POST', 'DELETE')
	
	# 初始化信息
	def __init__(self):
		self.url = None
		self.headers = dict()
		self.method = None
		self.params = {}
		self.data = {}
		self.response = Response()
		self.auth = None
		self.sent = False
	
	# repr 略过不提
	def __repr__(self):
		try:
			repr = '<Request [%s]>' % (self.method)
		except:
			repr = '<Request object>'
		return repr
	
	# 设置method时，会调用 __setattr__ 方法
	# 检查设置的值 是否在规定的方法 _METHODS 列表中
	# 若不在，则抛出 InvalidMethod 错误
	def __setattr__(self, name, value):
		if (name == 'method') and (value):
			if not value in self._METHODS:
				raise InvalidMethod()
		
		object.__setattr__(self, name, value)
	
	# 用于检查 url 是否设置
	# 若无设置，抛出 URLRequired 错误
	def _checks(self):
		"""Deterministic checks for consistiency."""

		if not self.url:
			raise URLRequired

	#  opener对象
	def _get_opener(self):
		""" Creates appropriate opener object for urllib2.
		"""
		
		# 如果需要 认证
		if self.auth:

			# create a password manager
			authr = urllib2.HTTPPasswordMgrWithDefaultRealm()

			authr.add_password(None, self.url, self.auth.username, self.auth.password)
			handler = urllib2.HTTPBasicAuthHandler(authr)
			opener = urllib2.build_opener(handler)

			# use the opener to fetch a URL
			return opener.open
		else:
			# 若无需认证
			return urllib2.urlopen
	
	。。。

Request类主要用于发送请求，因此重点关注其中的send方法，注释中解释了几点：

1. 发送请求，成功返回True，失败返回False
2. 如果传输过程中出错，则self.response.status_code会包含错误代码
3. 一旦请求成功发送，则Request类的sent属性会变为True
4. anyway参数若被设为True，则请求一定会被发送，不管是否曾发送过，

def send(self, anyway=False):
	"""Sends the request. Returns True of successfull, false if not.
	    If there was an HTTPError during transmission,
	    self.response.status_code will contain the HTTPError code.

	    Once a request is successfully sent, `sent` will equal True.
	
	    :param anyway: If True, request will be sent, even if it has
	    already been sent.
	"""
	self._checks()

	success = False
	
	if self.method in ('GET', 'HEAD', 'DELETE'):
           # 第一部分 ('GET', 'HEAD', 'DELETE')

	elif self.method == 'PUT':
           # 第二部分 PUT

	elif self.method == 'POST':
           # 第三部分 POST
	
	self.sent = True if success else False
	
	return success

在send中，会先进行self._checks()检查：

def _checks(self):
	"""Deterministic checks for consistiency."""

	if not self.url:
		raise URLRequired

这里只检测了URL是否设置，若没有则抛出URLRequired错误。然后根据method的不同分情况send请求，如果发送成功则success为True，sent变量也为True，然后返回success变量。

‘GET’, ‘HEAD’, ‘DELETE’

添加注释，代码如下：

def send(self, anyway=False):
	。。。

	if self.method in ('GET', 'HEAD', 'DELETE'):
		# 若不曾发送过 或者 不管任何情况
		if (not self.sent) or anyway:   

			# 如果 params是dict类型的话，进行urlencode
			# url encode GET params if it's a dict
			if isinstance(self.params, dict):
				params = urllib.urlencode(self.params)
			else:

				params = self.params

			# 获取 _Request 对象
			# :param ("%s?%s" % (self.url, params)): 组装url
			# :param method : 请求方法
			req = _Request(("%s?%s" % (self.url, params)), method=self.method)

			# 若有设置 headers 
			if self.headers:
				req.headers = self.headers

			# 获取 opener 对象 ，
			opener = self._get_opener()

			try:
				# 发出请求
				resp = opener(req) 
				# 状态码
				self.response.status_code = resp.code
				# 头部信息
				self.response.headers = resp.info().dict
				
				# 由于在这个判断分支中处理 'GET' 'HEAD', 'DELETE'三种请求
				# 'HEAD', 'DELETE' 并不是为了获取内容, 他们根据 status_code 即可判断是否请求成功
				# 若请求方法是 GET , 则设置返回的响应
				if self.method.lower() == 'get':
					#  设置响应的 content 值
					self.response.content = resp.read()
					
				# 请求成功,设置 success为 True
				success = True
			except urllib2.HTTPError, why:
				# 请求出错, 设置错误码
				self.response.status_code = why.code

‘PUT’

添加注释，代码如下：

def send(self, anyway=False):
	。。。
	# 请求方法为 PUT
	elif self.method == 'PUT':
		if (not self.sent) or anyway:
			# url 和 请求方法为PUT
			req = _Request(self.url, method='PUT')

			if self.headers:
				req.headers = self.headers

			# 设置PUT请求体
			req.data = self.data

			try:
				opener = self._get_opener()
				# 发处请求
				resp =  opener(req)
				# 设置响应
				self.response.status_code = resp.code
				self.response.headers = resp.info().dict
				self.response.content = resp.read()

				success = True

			except urllib2.HTTPError, why:
				self.response.status_code = why.code

‘POST’

添加注释，代码如下：

def send(self, anyway=False):
	。。。
	# 请求方法为 POST
	elif self.method == 'POST':
		if (not self.sent) or anyway
			
			# url 和 请求方法为POST
			req = _Request(self.url, method='POST')

			# 设置 headers
			if self.headers:
				req.headers = self.headers

			# 如果是dict的话，进行urlencode
			# url encode form data if it's a dict
			if isinstance(self.data, dict):
				req.data = urllib.urlencode(self.data)
			else:
				req.data = self.data

			try:
				# 获取opener
				opener = self._get_opener()

				# 发出请求
				resp =  opener(req)
				
				# 设置响应
				self.response.status_code = resp.code
				self.response.headers = resp.info().dict
				self.response.content = resp.read()

				success = True

			except urllib2.HTTPError, why:
				self.response.status_code = why.code

Response 类

在 Request类中我们见到在Request初始化__init__时设置了self.response = Response()。然后根据请求方法的不同，设置状态码self.response.status_code、响应头部self.response.headers、响应内容self.response.content 。接下来就看看response类是如何实现的。

class Response(object):
    """The :class:`Request` object. All :class:`Request` objects contain a
    :class:`Request.response <response>` attribute, which is an instance of
    this class.
    """

    def __init__(self):
        self.content = None
        self.status_code = None
        self.headers = dict()

    def __repr__(self):
        try:
            repr = '<Response [%s]>' % (self.status_code)
        except:
            repr = '<Response object>'
        return repr

AuthObject 类

该类暂时仅在 test_requests.py 中出现，用于设置认证的用户名和密码。代码如下：

class AuthObject(object):
    """The :class:`AuthObject` is a simple HTTP Authentication token. When
    given to a Requests function, it enables Basic HTTP Authentication for that
    Request. You can also enable Authorization for domain realms with AutoAuth.
    See AutoAuth for more details.s

    :param username: Username to authenticate with.
    :param password: Password for given username.
    """

    def __init__(self, username, password):
        self.username = username
        self.password = password

请求方法

get、post、put、delete和认证相关的方法 ，在代码结构上大同小异。

get

def get(url, params={}, headers={}, auth=None):
    """Sends a GET request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`.
    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
    """
	# 获取 Request对象
    r = Request()

	# 设置基本的请求参数
    r.method = 'GET'
    r.url = url
    r.params = params
    r.headers = headers
	# 设置认证信息
    r.auth = _detect_auth(url, auth)

	# 发起请求
    r.send()

	# 返回响应
    return r.response

head

def head(url, params={}, headers={}, auth=None):
    """Sends a HEAD request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`.
    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
    """
	# 获取 Request对象
    r = Request()

	# 设置基本信息
    r.method = 'HEAD'
    r.url = url
    # return response object
    r.params = params
    r.headers = headers
    r.auth = _detect_auth(url, auth)

	# 发处请求
    r.send()

	# 返回响应
    return r.response

post

def post(url, data={}, headers={}, auth=None):
    """Sends a POST request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Dictionary of POST Data to send with the :class:`Request`.
    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
    """
	# 获取Request对象
    r = Request()

	# 设置基本信息
    r.url = url
    r.method = 'POST'
    r.data = data

    r.headers = headers
    r.auth = _detect_auth(url, auth)

	# 发起请求
    r.send()

	# 返回响应
    return r.response

put

def put(url, data='', headers={}, auth=None):
    """Sends a PUT request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Bytes of PUT Data to send with the :class:`Request`.
    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
    """
	# 获取Request对象
    r = Request()

	# 设置基本信息
    r.url = url
    r.method = 'PUT'
    r.data = data

    r.headers = headers
    r.auth = _detect_auth(url, auth)

	# 发起请求
    r.send()

	# 返回响应
    return r.response

delete

def delete(url, params={}, headers={}, auth=None):
    """Sends a DELETE request. Returns :class:`Response` object.

    :param url: URL for the new :class:`Request` object.
    :param params: (optional) Dictionary of GET Parameters to send with the :class:`Request`.
    :param headers: (optional) Dictionary of HTTP Headers to sent with the :class:`Request`.
    :param auth: (optional) AuthObject to enable Basic HTTP Auth.
    """

	# 获取Request对象
    r = Request()

	# 设置基本信息
    r.url = url
    r.method = 'DELETE'
    # return response object

    r.headers = headers
    r.auth = _detect_auth(url, auth)

	# 发起请求
    r.send()

	# 返回响应
    return r.response

认证相关

从上面的请求方法实现中，可以发现有的请求带了如r.auth = _detect_auth(url, auth)

对于种种请求方法，我们不想在每次请求中都明确指出这次请求需不需要认证，但有些请求确实需要认证，因此在各种请求方法中都有一个可选参数auth=None，然后通过调用r.auth = _detect_auth(url, auth)来进一步设置。_detect_auth代码如下

def _detect_auth(url, auth):
    """Returns registered AuthObject for given url if available, defaulting to
    given AuthObject."""

    return _get_autoauth(url) if not auth else auth


def _get_autoauth(url):
    """Returns registered AuthObject for given url if available.
    """

    for (autoauth_url, auth) in AUTOAUTHS:
        if autoauth_url in url:
            return auth

    return None

对于明确指出需要认证的请求，自然auth参数也会指定。如果auth参数没有指定，则会调用_get_autoauth来查看是否有对应的规则。这个规则列表则由全局变量AUTOAUTHS来维护，如果请求的url包含autoauth_url，则返回autoauth_url对应的auth。如果不包含，则直接返回None。

为了维护这个全局变量AUTOAUTHS，它实现了一个add_autoauth方法如下：

def add_autoauth(url, authobject):
    """Registers given AuthObject to given URL domain. for auto-activation.
    Once a URL is registered with an AuthObject, the configured HTTP
    Authentication will be used for all requests with URLS containing the given
    URL string.

    Example: ::
        >>> c_auth = requests.AuthObject('kennethreitz', 'xxxxxxx')
        >>> requests.add_autoauth('https://convore.com/api/', c_auth)
        >>> r = requests.get('https://convore.com/api/account/verify.json')
        # Automatically HTTP Authenticated! Wh00t!

    :param url: Base URL for given AuthObject to auto-activate for.
    :param authobject: AuthObject to auto-activate.
    """
    global AUTOAUTHS

    AUTOAUTHS.append((url, authobject))

异常相关

不做过多解释。

class RequestException(Exception):
    """There was an ambiguous exception that occured while handling your request."""

class AuthenticationError(RequestException):
    """The authentication credentials provided were invalid."""

class URLRequired(RequestException):
    """A valid URL is required to make a request."""

class InvalidMethod(RequestException):
    """An inappropriate method was attempted."""