爬虫自学之路（二） requests小技巧

中文文档：http://docs.python-requests.org/zh_CN/latest/api.html
1 安装 pip install requests 或者 python -m install requests

2 RequestsCookieJar转换为cookie字典

requests.utils.dict_from_cookiejar(resp.cookies)

3 cookie字典转换为RequestsCookieJar

requests.utils.cookiejar_from_dict({"BDORZ":"27315"})

4 URL编码

requests.utils.quote("编码") # '%E7%BC%96%E7%A0%81'

5 URL解码

requests.utils.unquote("%E7%BC%96%E7%A0%81") # “编码”

6 不验证HTTPS

requests.get("https://www.12306.cn/mormhweb/", verify=False)

7 设置超时时间 和 retrying模块

requests.get("https://www.baidu.cn/", timeout=5)

8 retrying模块

import requests
from retrying import retry

headers={
"User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
"Referer": "https://m.douban.com/movie/",
}

@retry(stop_max_attempt_number=3)def _parse_url(url,method,data,proxies):
print("*"*20)
if method=="POST":
response = requests.post(url,data=data,headers=headers,proxies=proxies)
else:
response = requests.get(url,headers=headers,timeout=3,proxies=proxies)
assert response.status_code == 200
return response.content.decode()

def parse_url(url,method="GET",data=None,proxies={}):
try:
html_str = _parse_url(url,method,data,proxies)
except:
html_str = None

return html_str

if __name__ == '__main__':
url = "www.baidu.com"
print(parse_url(url))

9 urldecode 对字符串进行url解码
发起一个get请求时，会在url后携带参数