python - 爬虫之requests的初级使用详解（一）

最新推荐文章于 2024-07-12 16:16:27 发布

代码界的一股清流

最新推荐文章于 2024-07-12 16:16:27 发布

阅读量92

点赞数

分类专栏：采集数据基础（爬虫）文章标签： python 爬虫开发语言

本文链接：https://blog.csdn.net/li9320/article/details/132297358

版权

采集数据基础（爬虫）专栏收录该内容

2 篇文章 0 订阅

订阅专栏

总纲

1、requests发送请求（请求方法）

import requests

r = requests.get('https://api.github.com/events')
r = requests.options('https://api.github.com/events')
r = requests.head('https://api.github.com/events')
r = requests.post('https://api.github.com/events')
r = requests.put('https://api.github.com/events')
r = requests.patch('https://api.github.com/events')
r = requests.delete('https://api.github.com/events')
r = requests.request(method="get", url='https://api.github.com/events')

requests支持的请求方法：get、post、options、head、put、patch、delete六种。

2、传递url参数，各个参数详解

抠出源码 -- requests/api.pyi

from typing import Any

from .models import Response
from .sessions import RequestsCookieJar, _Auth, _Cert, _Data, _Files, _HooksInput, _Params, _TextMapping, _Timeout, _Verify

def request(
    method: str | bytes,
    url: str | bytes,
    *,
    params: _Params | None = ...,
    data: _Data | None = ...,
    headers: _TextMapping | None = ...,
    cookies: RequestsCookieJar | _TextMapping | None = ...,
    files: _Files | None = ...,
    auth: _Auth | None = ...,
    timeout: _Timeout | None = ...,
    allow_redirects: bool = ...,
    proxies: _TextMapping | None = ...,
    hooks: _HooksInput | None = ...,
    stream: bool | None = ...,
    verify: _Verify | None = ...,
    cert: _Cert | None = ...,
    json: Any | None = ...,
) -> Response: ...
def get(
    url: str | bytes,
    params: _Params | None = ...,
    *,
    data: _Data | None = ...,
    headers: _TextMapping | None = ...,
    cookies: RequestsCookieJar | _TextMapping | None = ...,
    files: _Files | None = ...,
    auth: _Auth | None = ...,
    timeout: _Timeout | None = ...,
    allow_redirects: bool = ...,
    proxies: _TextMapping | None = ...,
    hooks: _HooksInput | None = ...,
    stream: bool | None = ...,
    verify: _Verify | None = ...,
    cert: _Cert | None = ...,
    json: Any | None = ...,
) -> Response: ...
def options(
    url: str | bytes,
    *,
    params: _Params | None = ...,
    data: _Data | None = ...,
    headers: _TextMapping | None = ...,
    cookies: RequestsCookieJar | _TextMapping | None = ...,
    files: _Files | None = ...,
    auth: _Auth | None = ...,
    timeout: _Timeout | None = ...,
    allow_redirects: bool = ...,
    proxies: _TextMapping | None = ...,
    hooks: _HooksInput | None = ...,
    stream: bool | None = ...,
    verify: _Verify | None = ...,
    cert: _Cert | None = ...,
    json: Any | None = ...,
) -> Response: ...
def head(
    url: str | bytes,
    *,
    params: _Params | None = ...,
    data: _Data | None = ...,
    headers: _TextMapping | None = ...,
    cookies: RequestsCookieJar | _TextMapping | None = ...,
    files: _Files | None = ...,
    auth: _Auth | None = ...,
    timeout: _Timeout | None = ...,
    allow_redirects: bool = ...,
    proxies: _TextMapping | None = ...,
    hooks: _HooksInput | None = ...,
    stream: bool | None = ...,
    verify: _Verify | None = ...,
    cert: _Cert | None = ...,
    json: Any | None = ...,
) -> Response: ...
def post(
    url: str | bytes,
    data: _Data | None = ...,
    json: Any | None = ...,
    *,
    params: _Params | None = ...,
    headers: _TextMapping | None = ...,
    cookies: RequestsCookieJar | _TextMapping | None = ...,
    files: _Files | None = ...,
    auth: _Auth | None = ...,
    timeout: _Timeout | None = ...,
    allow_redirects: bool = ...,
    proxies: _TextMapping | None = ...,
    hooks: _HooksInput | None = ...,
    stream: bool | None = ...,
    verify: _Verify | None = ...,
    cert: _Cert | None = ...,
) -> Response: ...
def put(
    url: str | bytes,
    data: _Data | None = ...,
    *,
    params: _Params | None = ...,
    headers: _TextMapping | None = ...,
    cookies: RequestsCookieJar | _TextMapping | None = ...,
    files: _Files | None = ...,
    auth: _Auth | None = ...,
    timeout: _Timeout | None = ...,
    allow_redirects: bool = ...,
    proxies: _TextMapping | None = ...,
    hooks: _HooksInput | None = ...,
    stream: bool | None = ...,
    verify: _Verify | None = ...,
    cert: _Cert | None = ...,
    json: Any | None = ...,
) -> Response: ...
def patch(
    url: str | bytes,
    data: _Data | None = ...,
    *,
    params: _Params | None = ...,
    headers: _TextMapping | None = ...,
    cookies: RequestsCookieJar | _TextMapping | None = ...,
    files: _Files | None = ...,
    auth: _Auth | None = ...,
    timeout: _Timeout | None = ...,
    allow_redirects: bool = ...,
    proxies: _TextMapping | None = ...,
    hooks: _HooksInput | None = ...,
    stream: bool | None = ...,
    verify: _Verify | None = ...,
    cert: _Cert | None = ...,
    json: Any | None = ...,
) -> Response: ...
def delete(
    url: str | bytes,
    *,
    params: _Params | None = ...,
    data: _Data | None = ...,
    headers: _TextMapping | None = ...,
    cookies: RequestsCookieJar | _TextMapping | None = ...,
    files: _Files | None = ...,
    auth: _Auth | None = ...,
    timeout: _Timeout | None = ...,
    allow_redirects: bool = ...,
    proxies: _TextMapping | None = ...,
    hooks: _HooksInput | None = ...,
    stream: bool | None = ...,
    verify: _Verify | None = ...,
    cert: _Cert | None = ...,
    json: Any | None = ...,
) -> Response: ...

各个参数的含义：

method:请求方法
url: url地址
params: 查询字符串，默认为None
data: 表单数据或者json数据、默认为None
headers: 请求头信息,默认为None
cookies: cookie信息，默认为None
files:上传文件信息，默认为None
auth: 认证auth信息,
timeout:设置允许超时链接时间,默认None
allow_redirects:是否允许重定向,
proxies: 设置代理ip地址默认None
hooks:事件挂钩.
stream: 是否延迟下载.,
verify:是否认证SSL证书,
cert: 设置本地客户端 ...,
json: json数据 ...,

3、响应内容

1、文本响应内容

Requests 会自动解码来自服务器的内容。大多数 unicode 字符集都能被无缝地解码

>>> import requests
>>> r = requests.get('https://api.github.com/events')
>>> r.text
u'[{"repository":{"open_issues":0,"url":"https://github.com/...

2、二进制响应内容

你也能以字节的方式访问请求响应体，对于非文本请求：

>>> r.content
b'[{"repository":{"open_issues":0,"url":"https://github.com/...

3、json响应内容

Requests 中也有一个内置的 JSON 解码器，助你处理 JSON 数据

>>> import requests

>>> r = requests.get('https://api.github.com/events')
>>> r.json()
[{u'repository': {u'open_issues': 0, u'url': 'https://github.com/...

4、原始相应内容

在罕见的情况下，你可能想获取来自服务器的原始套接字响应，那么你可以访问 r.raw。如果你确实想这么干，那请你确保在初始请求中设置了 stream=True

>>> r = requests.get('https://api.github.com/events', stream=True)
>>> r.raw
<requests.packages.urllib3.response.HTTPResponse object at 0x101194810>
>>> r.raw.read(10)
'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03'

5、响应的其他属性

requests/model.pyi源码

class Response:
    __attrs__: Any
    _content: bytes | None  # undocumented
    status_code: int
    headers: CaseInsensitiveDict[str]
    raw: Any
    url: str
    encoding: str | None
    history: list[Response]
    reason: str
    cookies: RequestsCookieJar
    elapsed: datetime.timedelta
    request: PreparedRequest
    def __init__(self) -> None: ...
    def __bool__(self) -> bool: ...
    def __nonzero__(self) -> bool: ...
    def __iter__(self) -> Iterator[bytes]: ...
    def __enter__(self: Self) -> Self: ...
    def __exit__(self, *args: object) -> None: ...
    @property
    def next(self) -> PreparedRequest | None: ...
    @property
    def ok(self) -> bool: ...
    @property
    def is_redirect(self) -> bool: ...
    @property
    def is_permanent_redirect(self) -> bool: ...
    @property
    def apparent_encoding(self) -> str: ...
    def iter_content(self, chunk_size: int | None = ..., decode_unicode: bool = ...) -> Iterator[Any]: ...
    def iter_lines(
        self, chunk_size: int | None = ..., decode_unicode: bool = ..., delimiter: str | bytes | None = ...
    ) -> Iterator[Any]: ...
    @property
    def content(self) -> bytes: ...
    @property
    def text(self) -> str: ...
    def json(
        self,
        *,
        cls: type[JSONDecoder] | None = ...,
        object_hook: Callable[[dict[Any, Any]], Any] | None = ...,
        parse_float: Callable[[str], Any] | None = ...,
        parse_int: Callable[[str], Any] | None = ...,
        parse_constant: Callable[[str], Any] | None = ...,
        object_pairs_hook: Callable[[list[tuple[Any, Any]]], Any] | None = ...,
        **kwds: Any,
    ) -> Any: ...
    @property
    def links(self) -> dict[Any, Any]: ...
    def raise_for_status(self) -> None: ...
    def close(self) -> None: ...

4、请求 -- 定制请求头headers参数。

>>> url = 'https://api.github.com/some/endpoint'
>>> headers = {'user-agent': 'my-app/0.0.1'}

>>> r = requests.get(url, headers=headers)
注意事项：

如果在 .netrc 中设置了用户认证信息，使用 headers= 设置的授权就不会生效。而如果设置了 auth= 参数，``.netrc`` 的设置就无效了。
如果被重定向到别的主机，授权 header 就会被删除。
代理授权 header 会被 URL 中提供的代理身份覆盖掉。
在我们能判断内容长度的情况下，header 的 Content-Length 会被改写
所有的 header 值必须是 string、bytestring 或者 unicode。尽管传递 unicode header 也是允许的，但不建议这样做。

5、请求 -- POST请求的data参数

#  表单字典形式
>>> payload = {'key1': 'value1', 'key2': 'value2'}
>>> r = requests.post("http://httpbin.org/post", data=payload)


# 元组列表
# 你还可以为 data 参数传入一个元组列表。在表单中多个元素使用同一 key 的时候
>>> payload = (('key1', 'value1'), ('key1', 'value2'))
>>> r = requests.post('http://httpbin.org/post', data=payload)


# json数据
>>> payload = {'some': 'data'}
>>> r = requests.post(url, data=json.dumps(payload))

6、请求 -- POST请求的file参数上传文件

Requests 使得上传多部分编码文件变得很简单：
>>> files = {'file': open('report.xls', 'rb')}

>>> r = requests.post(url, files=files)
你可以显式地设置文件名，文件类型和请求头：
>>> files = {'file': ('report.xls', open('report.xls', 'rb'), 'application/vnd.ms-excel', {'Expires': '0'})}

>>> r = requests.post(url, files=files)
你也可以发送作为文件来接收的字符串，
files = {'file': ('report.csv', 'some,data,to,send\nanother,row,to,send\n')}

>>> r = requests.post(url, files=files)

7、请求 -- 设置cookie信息

url = 'http://httpbin.org/cookies'
>>> cookies = dict(cookies_are='working')

>>> r = requests.get(url, cookies=cookies)

8、请求 -- 是否允许重定向

 r = requests.head('http://github.com', allow_redirects=True) # 启动
 r = requests.head('http://github.com', allow_redirects=False) # 禁用

9、请求 -- 设置超时时间

requests.get('http://github.com', timeout=0.001)
注意：timeout 仅对连接过程有效，与响应体的下载无关

10、响应 -- 响应状态码

我们可以检测响应状态码

>>> r = requests.get('http://httpbin.org/get')
>>> r.status_code
200

Requests还附带了一个内置的状态码查询对象。

>>> r.status_code == requests.codes.ok
True

codes包含的状态码（requests/status_codes.py）

_codes = {

    # Informational.
    100: ('continue',),
    101: ('switching_protocols',),
    102: ('processing',),
    103: ('checkpoint',),
    122: ('uri_too_long', 'request_uri_too_long'),
    200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'),
    201: ('created',),
    202: ('accepted',),
    203: ('non_authoritative_info', 'non_authoritative_information'),
    204: ('no_content',),
    205: ('reset_content', 'reset'),
    206: ('partial_content', 'partial'),
    207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'),
    208: ('already_reported',),
    226: ('im_used',),

    # Redirection.
    300: ('multiple_choices',),
    301: ('moved_permanently', 'moved', '\\o-'),
    302: ('found',),
    303: ('see_other', 'other'),
    304: ('not_modified',),
    305: ('use_proxy',),
    306: ('switch_proxy',),
    307: ('temporary_redirect', 'temporary_moved', 'temporary'),
    308: ('permanent_redirect',
          'resume_incomplete', 'resume',),  # These 2 to be removed in 3.0

    # Client Error.
    400: ('bad_request', 'bad'),
    401: ('unauthorized',),
    402: ('payment_required', 'payment'),
    403: ('forbidden',),
    404: ('not_found', '-o-'),
    405: ('method_not_allowed', 'not_allowed'),
    406: ('not_acceptable',),
    407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'),
    408: ('request_timeout', 'timeout'),
    409: ('conflict',),
    410: ('gone',),
    411: ('length_required',),
    412: ('precondition_failed', 'precondition'),
    413: ('request_entity_too_large',),
    414: ('request_uri_too_large',),
    415: ('unsupported_media_type', 'unsupported_media', 'media_type'),
    416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'),
    417: ('expectation_failed',),
    418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'),
    421: ('misdirected_request',),
    422: ('unprocessable_entity', 'unprocessable'),
    423: ('locked',),
    424: ('failed_dependency', 'dependency'),
    425: ('unordered_collection', 'unordered'),
    426: ('upgrade_required', 'upgrade'),
    428: ('precondition_required', 'precondition'),
    429: ('too_many_requests', 'too_many'),
    431: ('header_fields_too_large', 'fields_too_large'),
    444: ('no_response', 'none'),
    449: ('retry_with', 'retry'),
    450: ('blocked_by_windows_parental_controls', 'parental_controls'),
    451: ('unavailable_for_legal_reasons', 'legal_reasons'),
    499: ('client_closed_request',),

    # Server Error.
    500: ('internal_server_error', 'server_error', '/o\\', '✗'),
    501: ('not_implemented',),
    502: ('bad_gateway',),
    503: ('service_unavailable', 'unavailable'),
    504: ('gateway_timeout',),
    505: ('http_version_not_supported', 'http_version'),
    506: ('variant_also_negotiates',),
    507: ('insufficient_storage',),
    509: ('bandwidth_limit_exceeded', 'bandwidth'),
    510: ('not_extended',),
    511: ('network_authentication_required', 'network_auth', 'network_authentication'),
}

11、响应 -- 响应头信息

我们可以查看以一个 Python 字典形式展示的服务器响应头：

>>> r.headers
{
    'content-encoding': 'gzip',
    'transfer-encoding': 'chunked',
    'connection': 'close',
    'server': 'nginx/1.0.4',
    'x-runtime': '148ms',
    'etag': '"e1ca502697e5c9317743dc078f67693f"',
    'content-type': 'application/json'
}

12、响应 -- 响应的cookie信息

>>> url = 'http://example.com/some/cookie/setting/url'
>>> r = requests.get(url)

>>> r.cookies['example_cookie_name']

13、响应 -- 请求的历史信息

>>> r = requests.get('http://github.com')

>>> r.url
'https://github.com/'

>>> r.status_code
200

>>> r.history
[<Response [301]>]

14、错误与异常

requests/exception.py源码异常种类

# -*- coding: utf-8 -*-

"""
requests.exceptions
~~~~~~~~~~~~~~~~~~~

This module contains the set of Requests' exceptions.
"""
from urllib3.exceptions import HTTPError as BaseHTTPError

from .compat import JSONDecodeError as CompatJSONDecodeError


class RequestException(IOError):
    """There was an ambiguous exception that occurred while handling your
    request.
    """

    def __init__(self, *args, **kwargs):
        """Initialize RequestException with `request` and `response` objects."""
        response = kwargs.pop('response', None)
        self.response = response
        self.request = kwargs.pop('request', None)
        if (response is not None and not self.request and
                hasattr(response, 'request')):
            self.request = self.response.request
        super(RequestException, self).__init__(*args, **kwargs)


class InvalidJSONError(RequestException):
    """A JSON error occurred."""


class JSONDecodeError(InvalidJSONError, CompatJSONDecodeError):
    """Couldn't decode the text into json"""


class HTTPError(RequestException):
    """An HTTP error occurred."""


class ConnectionError(RequestException):
    """A Connection error occurred."""


class ProxyError(ConnectionError):
    """A proxy error occurred."""


class SSLError(ConnectionError):
    """An SSL error occurred."""


class Timeout(RequestException):
    """The request timed out.

    Catching this error will catch both
    :exc:`~requests.exceptions.ConnectTimeout` and
    :exc:`~requests.exceptions.ReadTimeout` errors.
    """


class ConnectTimeout(ConnectionError, Timeout):
    """The request timed out while trying to connect to the remote server.

    Requests that produced this error are safe to retry.
    """


class ReadTimeout(Timeout):
    """The server did not send any data in the allotted amount of time."""


class URLRequired(RequestException):
    """A valid URL is required to make a request."""


class TooManyRedirects(RequestException):
    """Too many redirects."""


class MissingSchema(RequestException, ValueError):
    """The URL scheme (e.g. http or https) is missing."""


class InvalidSchema(RequestException, ValueError):
    """The URL scheme provided is either invalid or unsupported."""


class InvalidURL(RequestException, ValueError):
    """The URL provided was somehow invalid."""


class InvalidHeader(RequestException, ValueError):
    """The header value provided was somehow invalid."""


class InvalidProxyURL(InvalidURL):
    """The proxy URL provided is invalid."""


class ChunkedEncodingError(RequestException):
    """The server declared chunked encoding but sent an invalid chunk."""


class ContentDecodingError(RequestException, BaseHTTPError):
    """Failed to decode response content."""


class StreamConsumedError(RequestException, TypeError):
    """The content for this response was already consumed."""


class RetryError(RequestException):
    """Custom retries logic failed"""


class UnrewindableBodyError(RequestException):
    """Requests encountered an error when trying to rewind a body."""

# Warnings


class RequestsWarning(Warning):
    """Base warning for Requests."""


class FileModeWarning(RequestsWarning, DeprecationWarning):
    """A file was opened in text mode, but Requests determined its binary length."""


class RequestsDependencyWarning(RequestsWarning):
    """An imported dependency doesn't match the expected version range."""

本文借鉴requests模块官方文档：

Requests: 让 HTTP 服务人类¶

代码界的一股清流

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
python - 爬虫之requests的初级使用详解（一）

requests支持的请求方法：get、post、options、head、put、patch、delete六种。Requests 中也有一个内置的 JSON 解码器，助你处理 JSON 数据。在罕见的情况下，你可能想获取来自服务器的原始套接字响应，那么你可以访问。data: 表单数据或者json数据、默认为None。cookies: cookie信息，默认为None。headers: 请求头信息,默认为None。params: 查询字符串，默认为None。files:上传文件信息，默认为None。
复制链接

扫一扫