爬虫之request模块介绍使用

1.request模块安装

（1）cmd中pip install requests
（2）开发者工具安装

• pycharm 里面的Terminal安装
  -
• pycharm的file - setting -project - + - 输入要安装的包
  

2.requests模块

2.1 常用方法

常用方法为requests.get()方法
requests.get(url,headers=headers)

import requests
# 发生请求,获取响应
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36','Cookie':'PSTM=1582094483; BAIDUID=8F990E30034CDA3119553480B495C931:FG=1; BIDUPSID=B663A1F6E460EBCEB8F650086AA076DA; BD_UPN=12314753; BDUSS=V1WlhRN1NKUDhlaW81a1A2T0FIZTZFbWRHVGpEaDNpSDlvWDdzVTR3OC1CblplSVFBQUFBJCQAAAAAAAAAAAEAAACuKFyEzM7MznJ5YW5zdW5ueQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD55Tl4-eU5eVk; H_WISE_SIDS=141750_142978_143219_142067_142113_141182_143858_132921_143452_142697_143133_142210_143879_141875_140631_139051_141745_143789_142511_139172_141901_142779_131246_137749_136687_138883_140259_141942_127969_142874_140066_142991_140593_134047_143059_143492_139550_140350_138426_143469_143276_141930_131423_107312_138596_139884_140974_143477_142911_140312_140798_141364_110085; MSA_WH=1280_578; delPer=0; BD_CK_SAM=1; BDRCVFR[CDku5Bol_IT]=mbxnW11j9Dfmh7GuZR8mvqV; BD_HOME=1; PSINO=6; ZD_ENTRY=baidu; cflag=13%3A3; H_PS_PSSID=31356_1452_31325_21121_31422_31341_31464_31228_30824_26350_31164_31475_22159; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; COOKIE_SESSION=163428_0_5_3_7_2_0_0_4_1_0_0_251559_0_9_0_1588489645_0_1588489636%7C9%23151359_19_1586660480%7C8; H_PS_645EC=08d162EIZDw481CZm5OsrPcbvOAkGXMU3fUIElgyDj6qYNIjiDvSbVj1rlU'}
url = 'https://www.baidu.com/s?'  # 百度网址
response = requests.get(url,headers=headers)
print(response)  # 结果为：<Response [200]>

2.2响应对象的方法

requests.get(url,params=wd,headers=headers)
response.content.decode(encoding=‘utf-8’)

# 发生请求，请求响应
# https://www.baidu.com/s?wd=%E4%B8%AD%E5%9B%BD
wd = {'wd':'中国'}
response = requests.get(url,params=wd,headers=headers)  # urllib用三个命令，urllib.request.Request、urllib.open、response.read.decode
html = response.text  # 属性  返回的是一个str类型的数据
# print(html)  # 得到内容未解码
response.encoding = 'utf-8'
html = response.text  # 解码

html = response.content  # 返回的是一个字节流
html = response.content.decode(encoding='utf-8')  # 解码
print(html)  # 得到内容未解码

2.3发送post请求

response = requests.post(url,data=data,headers=headers)

import requests
key = input('请输入您要翻译的内容:')
data = {
    'i': key,
    'from': 'AUTO',
    'smartresult': 'dict',
    'client': 'fanyideskweb',
    'salt': '15880623642174',
    'sign': 'c6c2e897040e6cbde00cd04589e71d4e',
    'ts': '1588062364217',
    'bv': '42160534cfa82a6884077598362bbc9d',
    'doctype': 'json',
    'version': '2.1',
    'keyfrom':'fanyi.web',
    'action': 'FY_BY_CLICKBUTTION'
}
url = 'http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'  # 注意此处的url是Request URL，并且需要把_o去掉才能实现，否则返回的html是{"errorCode":50}
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'}
res = requests.post(url,data=data,headers=headers) # 采用post方法
res.encoding = 'utf-8'
html = res.text
r_dict = json.loads(html)
result = r_dict['translateResult'][0][0]['tgt']
print(result)

2.4requests设置代理

proxy = { ‘http’:‘36.248.129.239:9999’}
res = requests.get(url,proxies=proxy)

# requests 设置代理
# 使⽤requests添加代理只需要在请求⽅法中(get/post)传递proxies参数就可以了
# 免费的代理网址为：⻄刺免费代理IP：http://www.xicidaili.com/ 快代理：http://www.kuaidaili.com/ 

proxy = { 'http':'36.248.129.239:9999'}
url = 'http://www.httpbin.org/ip'
res = requests.get(url,proxies=proxy)  # 以IP地址为'36.248.129.239:9999'进行访问
res.encoding = 'utf-8'
print(res.text)

2.5cookie

cookie ：通过在客户端记录的信息确定用户身份
HTTP是⼀种无连接协议,客户端和服务器交互仅仅限于 请求/响应过程,结束后断开,下⼀次请求时,服务器会认为是⼀个新的客户端,为了维护他们之间的连接,让服务器知道这是前⼀个用户发起的请求,必须在⼀个地方保存客户端信息（必须在⼀个地方的地方指代的是cookie）。

resp.cookies，resp.cookies.get_dict()获取cookie值
headers里面添加cookie设置实现登录和会话维持

# requests.cookie()
# cookie保持一个登录的状态，进行一个会话的维持
import requests
resp = requests.get('https://www.baidu.com/')
print(resp.cookies)  # <RequestsCookieJar[<Cookie BDORZ=27315 for .baidu.com/>]>
print(resp.cookies.get_dict()  # {'BDORZ': '27315'}

#模拟登陆知乎
url = 'https://www.zhihu.com/hot'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36'}
resp = requests.get(url,headers=headers)
print(resp.text)  # 无法登录，因为headers里面没有录入cookie数据
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36','cookie':'_zap=f6651dfd-8259-4706-9032-5727ec6593ff; d_c0="AKAWpA4b6BCPTrYOvjRlh-tSAC2xRRy2R_o=|1583234256"; _ga=GA1.2.1237704894.1583234257; _xsrf=EQmHq5EuP5gF6Ja6bH46i3znv0r53niY; _gid=GA1.2.1825342243.1588076980; tst=h; tshl=; Hm_lvt_98beee57fd2ef70ccdd5ca52b9740c49=1587811271,1588076979,1588228873,1588246738; SESSIONID=aq5YCH9MiITrFZOobkIFT3EYgtlfG6SlvGwVB2EUB1F; JOID=UFwUAkLNy7aYh4WBEc5mLyDPZL4Dqr-Dyc_LvVyvhOfqydTIe7wBFMWKhoQZq-aJtgz8-vsmayVtXOxAwCJS2b4=; osd=UlgUC0zPz7aRiYeFEcdoLSTPbbABrr-Kx83PvVWhhuPqwNrKf7wIGseOho0XqeKJvwL-_vsvZSdpXOVOwiZS0LA=; capsion_ticket="2|1:0|10:1588254120|14:capsion_ticket|44:Yjk0ZTgyMjRjZDU0NGFlMjgwMzU4ZmZkMWJhYzA5MmI=|fdf13162982002c673847fae50e99c8f22d583ef7e23228c2d3ace7080b56ee7"; z_c0="2|1:0|10:1588254121|4:z_c0|92:Mi4xRjdYeENBQUFBQUFBb0Jha0Rodm9FQ1lBQUFCZ0FsVk5xU09ZWHdEcnRjZFhPSlkwdXpYZXFualQtekloamplbzdn|76d278afd875611d83dba20ed4d6169d34d0bf1447521478b93ec7ec38c443ae"; Hm_lpvt_98beee57fd2ef70ccdd5ca52b9740c49=1588254123; KLBRSID=ca494ee5d16b14b649673c122ff27291|1588254207|1588252528'}
resp = requests.get(url,headers=headers)
print(resp.text)  # 实现登录，因为headers录入了cookie数据

2.6session

session ：通过在服务端记录的信息确定用户身份，这里这个session就是⼀个指的是会话

2.7处理不信任的SSL证书

（1）什么是SSL证书？

• SSL证书是数字证书的⼀种，类似于驾驶证、护照和营业执照的电⼦副本。因为配置在服务器上，也称为SSL服务器证书。
• SSL 证书就是遵守 SSL协议，由受信任的数字证书颁发机构CA，在验证服务器身份后颁发，具有服务器身份验证和数据传输加密功能.
  例如：下面网址为不可信任网址，则设置为requests.get(url,verify = False)，进行访问
  

# 处理不信任的SSL证书
import requests
url = 'https://inv-veri.chinatax.gov.cn/'

# resp = requests.get(url)
# print(resp.text)  # 报错SSLerror

resp = requests.get(url,verify = False)  # 设定为不认证SSL证书
print(resp.text)