1.2、数据请求—requests库

最新推荐文章于 2023-02-07 15:20:25 发布

TUJC

最新推荐文章于 2023-02-07 15:20:25 发布

阅读量929

点赞数

分类专栏： python爬虫

本文链接：https://blog.csdn.net/TU_JCN/article/details/98522955

版权

python爬虫专栏收录该内容

11 篇文章 0 订阅

订阅专栏

import requests

class RequestSpider(object):
    def __init__(self):
        url = 'https://www.baidu.com'
        headers = {
             'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
        }
        self.response = requests.get(url, headers=headers)

    def run(self):

        data = self.response.content

        # 获取请求头
        request_headers = self.response.request.headers

        # 获取响应头
        coderesponse_headers = self.response.headers

        # 响应状态码
        code = self.response.status_code

        # 请求的cookie
        request_cookie = self.response.request._cookies
        print(request_cookie)

        # 响应的cookie
        response_cookie = self.response.cookies
        print(response_cookie)

RequestSpider().run()

一、GET请求

requests.get(url, params=None, **kwargs)
   
    :param url: URL for the new :class:`Request` object.
    :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response

1. 最基本的GET请求可以直接用get方法

import requests


response = requests.get("http://www.baidu.com/")

# 也可以这么写
response = requests.request("get", "http://www.baidu.com/")



# 内身份认证 auth 

response = requests.get(url,auth=('username','password'))

print(response.status_code)


# content属性 返回的类型 是bytes
data = response.content.decode('utf-8')

# text 属性 返回的类型 是文本str
data = response.text

print(type(data))

2. 添加 headers 和查询参数

如果想添加 headers，可以传入headers参数来增加请求头中的headers信息。

如果要将参数放在url中传递，可以利用 params 参数。

params参数自动转译，自动转为url编码，不需urlencode（）

import requests

# url = 'https://www.baidu.com/s?wd=美女'
url = 'https://www.baidu.com/s'
params = {
    'wd':"美女"
}

headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
}

response = requests.get(url,headers=headers, params=params)
# params参数 自动转译,自动转为url编码，不需urlencode（）

data = response.content.decode()


with open('baidu.html', 'w') as f:
    f.write(data)

# 发送post 和添加参数
requests.post(url,data=(参数{}),json=(参数))

# https://www.baidu.com/s?wd=%E7%BE%8E%E5%A5%B3&rsv_spt=1&rsv_iqid=0xefb8b43600013949&issp=1&f=8&rsv_bp=1&rsv_idx=2&ie=utf-8&rqlang=cn&tn=baiduhome_pg&rsv_enter=1&oq=%25E5%25A4%25B4%25E6%259D%25A1&rsv_t=6e3aSjYtw0WgEg7MAIuUlOc3D5lwFBJUVw3KsdkhkWYhZWcNMn9kLBO12GflHlOeUHxx&inputT=506&rsv_pq=81d8f9470001b348&rsv_sug3=19&rsv_sug1=16&rsv_sug7=100&bs=%E5%A4%B4%E6%9D%A1

import requests
import json


url = 'https://api.github.com/user'
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
}

# 这个 网址 返回的内容不是html 而是标准的json
response = requests.get(url, headers=headers)

# str
# data = response.content.decode()
# # str-- dict
# data_dict = json.loads(data)


# json() 自动将json字符串 转换成Python dict list
data = response.json()

print(data['message'])

2、POST请求

requests.post(url, data=None, json=None, **kwargs)
  
    
    :param url: URL for the new :class:`Request` object.
    :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`.
    :param json: (optional) json data to send in the body of the :class:`Request`.
    :param \*\*kwargs: Optional arguments that ``request`` takes.
    :return: :class:`Response <Response>` object
    :rtype: requests.Response

1. 最基本的GET请求可以直接用post方法

import requests

# 发送post请求

url=''
data = {

}
response = requests.post(url, data=data)

2. 传入data数据

对于 POST 请求来说，我们一般需要为它增加一些参数。那么最基本的传参方法可以利用 data 这个参数。

import requests

formdata = {
    "type":"AUTO",
    "i":"i love python",
    "doctype":"json",
    "xmlVersion":"1.8",
    "keyfrom":"fanyi.web",
    "ue":"UTF-8",
    "action":"FY_BY_ENTER",
    "typoResult":"true"
}

url = "http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule&smartresult=ugc&sessionFrom=null"

headers={ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}

response = requests.post(url, data = formdata, headers = headers)

print(response.text)

# 如果是json文件可以直接显示
print(response.json())

3、代理设置

如果需要使用代理，你可以通过为任意请求方法提供 proxies 参数来配置单个请求：

私密代理

import requests

# 1.请求url
url = 'http://www.baidu.com'
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
}

free_proxy = {'http': '27.17.45.90:43411'}

response = requests.get(url=url, headers=headers, proxies=free_proxy)

print(response.status_code)

web客户端验证

如果是Web客户端验证，需要添加 auth = (账户名, 密码)

import requests

auth=('test', '123456')

response = requests.get('http://192.168.199.107', auth = auth)

print response.text

4、SSL 证书验证

设置 verify=False

import requests

url = 'https://www.12306.cn/mormhweb/'
headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
}

# 因为hhtps  是有第三方 CA 证书认证的
# 但是 12306  虽然是https 但是 它不是 CA证书, 他是自己 颁布的证书
# 解决方法 是: 告诉 web 忽略证书 访问
# 设置  verify=False

response = requests.get(url=url, headers=headers, verify=False)
data = response.content.decode()

with open('03-ssl.html', 'w') as f:
    f.write(data)

# requests.exceptions.SSLError: HTTPSConnectionPool(host=

5、Cookies

方法1：手动提取cookie，登录

import requests

# 请求数据url
member_url = 'https://www.yaozh.com/member/'

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
}
#  cookies 的字符串
cookies = '_ga=GA1.2.1820447474.1535025127; MEIQIA_EXTRA_TRACK_ID=199Tty9OyANCXtHaSobJs67FU7J; WAF_SESSION_ID=7d88ae0fc48bffa022729657cf09807d; PHPSESSID=70kadg2ahpv7uuc8docd09iat4; _gid=GA1.2.133568065.1540383729; _gat=1; MEIQIA_VISIT_ID=1C1OdtdqpgpGeJ5A2lCKLMGiR4b; yaozh_logintime=1540383753; yaozh_user=381740%09xiaomaoera12; yaozh_userId=381740; db_w_auth=368675%09xiaomaoera12; UtzD_f52b_saltkey=ylH82082; UtzD_f52b_lastvisit=1540380154; UtzD_f52b_lastact=1540383754%09uc.php%09; UtzD_f52b_auth=f958AVKmmdzQ2CWwmr6GMrIS5oKlW%2BkP5dWz3SNLzr%2F1b6tOE6vzf7ssgZDjhuXa2JsO%2FIWtqd%2FZFelWpPHThohKQho; yaozh_uidhas=1; yaozh_mylogin=1540383756; MEIQIA_EXTRA_TRACK_ID=199Tty9OyANCXtHaSobJs67FU7J; WAF_SESSION_ID=7d88ae0fc48bffa022729657cf09807d; Hm_lvt_65968db3ac154c3089d7f9a4cbb98c94=1535025126%2C1535283389%2C1535283401%2C1539351081%2C1539512967%2C1540209934%2C1540383729; MEIQIA_VISIT_ID=1C1OdtdqpgpGeJ5A2lCKLMGiR4b; Hm_lpvt_65968db3ac154c3089d7f9a4cbb98c94=1540383761'

# 需要的是 字典类型
cook_dict = {}
cookies_list = cookies.split('; ')
for cookie in cookies_list:
    cook_dict[cookie.split('=')[0]] = cookie.split('=')[1]


# 字典推导式
cook_dict = {cookie.split('=')[0]:cookie.split('=')[1] for cookie in cookies.split('; ')}

response = requests.get(member_url, headers=headers, cookies=cook_dict)

data = response.content.decode()

with open('05-cookie.html','w') as f:
    f.write(data)

方法二：利用会话session维持，自动获取cookie

import requests

# 请求数据url
member_url = 'https://www.yaozh.com/member/'

headers = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.67 Safari/537.36'
}
# session 类 可以自动保存cookies === cookiesJar

session = requests.session()

# 1.代码登录
login_url = 'https://www.yaozh.com/login'
login_form_data = {
    'username':'xiaomaoera12',
    'pwd': 'lina081012',
    'formhash': '54AC1EE419',
    'backurl': 'https%3A%2F%2Fwww.yaozh.com%2F',
}

login_response = session.post(login_url,data=login_form_data,headers=headers)
print(login_response.content.decode())

# 2.登录成功之后 带着 有效的cookies 访问 请求目标数据
data = session.get(member_url,headers=headers).content.decode()

with open('05-cookie2.html','w') as f:
    f.write(data)