Requests是用python语言基于urllib编写的,采用的是Apache2 Licensed开源协议的HTTP库。与urllib相比,Requests更加方便,可以节约我们大量的工作,建议爬虫使用Requests库。
1. 发起基本get请求并传入参数
import requests
def get_html(url):
# 这个传入的参数是拼接在url中的一组参数
param = {"wd": "c++"}
html = requests.get(url,params=param)
if html.status_code == 200:
html.encoding="utf8"
print(html.text)
else:
print("ERROR",html)
if __name__ == '__main__':
url = "http://www.baidu.com/s"
get_html(url)
2.发起post请求并提交请求体
# post发送表单
import requests
def post_html(url):
data = {"name": "python", "pwd": "123"}
html = requests.post(url, data=data)
if html.status_code == 200:
html.encoding = "utf8"
print(html.text)
else:
print("get error:" + html.url)
if __name__ == '__main__':
url = "http://httpbin.org/post"
post_html(url)
3.获取json数据
# 获取json数据
import requests
def json_html(url):
html = requests.get(url)
if html.status_code == 200:
html.encoding = "utf8"
for news in html.json():
print(news['id'],news['title'])
else:
print("get error:" + html.url)
if __name__ == '__main__':
url = "https://news.qq.com/ext2020/apub/json/prevent.new.json"
json_html(url)
4.图片下载
import requests
from uuid import uuid4
# 图片下载
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
" AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36",
"Referer": "https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&"
"st=-1&fr=&sf=1&fmq=1567133149621_R&pv=&ic=0&nc=1&z=0&hd=0&latest=0©right=0&se"
"=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&sid=&word=%E5%A3%81%E7%BA%B8",
"Cookie": "BIDUPSID=A9CA40CD7509DCDBCB2358D36CBB2D91; BAIDUID=E0E4E5065F8DB6020266232584DB75EE:FG=1;"
" PSTM=1595728572; BDUSS=BwUW5IRjlzcHhLV3E0azBpbkdjTHpGbHhIejFLMG1XanIyY3pJTEVDLXUtVVJmRVF"
"BQUFBJCQAAAAAAAAAAAEAAAB4xj02QWxwYWNhZGFoYdi8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
"AAAAAAAAAAAAAAAAAAAAAAAAAK5sHV-ubB1ffl; BDRCVFR[xoix5KwSHTc]=9xWipS8B-FspA7EnHc1QhPEUf; de"
"lPer=0; PSINO=2; H_PS_PSSID=; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BDRCVFR[X_XKQks0S63]"
"=mk3SLVN4HKm; userFrom=www.baidu.com; BDRCVFR[-pGxjrCMryR]=mk3SLVN4HKm; firstShowTip=1"
}
def post_url(url):
html = requests.get(url, headers=headers)
if html.status_code == 200:
html.encoding = "utf8"
content = html.json()['data']
# print(content)
for item in content:
if len(item) > 0:
download(item['middleURL'])
def download(url):
img = requests.get(url, headers=headers)
print("正在下载{}".format(url))
with open("./images/{}.jpg".format(uuid4()), 'wb') as f:
# 每次下载225字节
for chunk in img.iter_content(225):
f.write(chunk)
if __name__ == '__main__':
url = "https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A3%81%E7%BA%B8&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=0&ic=0&hd=0&latest=0©right=0&word=%E5%A3%81%E7%BA%B8&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&expermode=&force=&cg=wallpaper&pn=30&rn=30&gsm=1e&1595855122224="
post_url(url)
5.使用session维持会话
# 使用session维持会话,减少频繁发起请求
import requests
from uuid import uuid4
# 图片下载
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
" AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36",
"Referer": "https://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&"
"st=-1&fr=&sf=1&fmq=1567133149621_R&pv=&ic=0&nc=1&z=0&hd=0&latest=0©right=0&se"
"=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&sid=&word=%E5%A3%81%E7%BA%B8",
"Cookie": "BIDUPSID=A9CA40CD7509DCDBCB2358D36CBB2D91; BAIDUID=E0E4E5065F8DB6020266232584DB75EE:FG=1;"
" PSTM=1595728572; BDUSS=BwUW5IRjlzcHhLV3E0azBpbkdjTHpGbHhIejFLMG1XanIyY3pJTEVDLXUtVVJmRVF"
"BQUFBJCQAAAAAAAAAAAEAAAB4xj02QWxwYWNhZGFoYdi8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
"AAAAAAAAAAAAAAAAAAAAAAAAAK5sHV-ubB1ffl; BDRCVFR[xoix5KwSHTc]=9xWipS8B-FspA7EnHc1QhPEUf; de"
"lPer=0; PSINO=2; H_PS_PSSID=; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; BDRCVFR[X_XKQks0S63]"
"=mk3SLVN4HKm; userFrom=www.baidu.com; BDRCVFR[-pGxjrCMryR]=mk3SLVN4HKm; firstShowTip=1"
}
session = requests.session()
session.headers = headers
def post_url(url):
html = session.get(url)
if html.status_code == 200:
html.encoding = "utf8"
content = html.json()['data']
# print(content)
for item in content:
if len(item) > 0:
download(item['middleURL'])
def download(url):
img = session.get(url)
print("正在下载{}".format(url))
with open("./images/{}.jpg".format(uuid4()), 'wb') as f:
# 每次下载225字节
for chunk in img.iter_content(225):
f.write(chunk)
if __name__ == '__main__':
url = "https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E5%A3%81%E7%BA%B8&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=0&ic=0&hd=0&latest=0©right=0&word=%E5%A3%81%E7%BA%B8&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&expermode=&force=&cg=wallpaper&pn=30&rn=30&gsm=1e&1595855122224="
post_url(url)
6.忽略https证书验证
# 忽略https证书验证
import requests
def get_url(url):
html = requests.get(url,verify=False)
if html.status_code == 200:
print("OK")
else:
print("Error:" + url)
if __name__ == '__main__':
url = "https://kennethreitz.org"
get_url(url)
7.使用IP代理
# 使用IP代理
import requests
proxies = {
"http": "http://118.212.104.230:9999",
"https": "http://171.35.221.59:9000"
}
def get_url(url):
html = requests.get(url, proxies=proxies, timeout=3,verify=False)
if html.status_code == 200:
print("ok")
else:
print("get error:" + url)
if __name__ == '__main__':
url = "https://kennethreitz.org"
get_url(url)
8.post上传文件
# post上传文件
import requests
file = {
'file': ('demo1', open('demo1.py', 'rb'), 'application/html', {'Expires': '0'})
}
def post_html(url):
html = requests.post(url, files=file)
if html.status_code == 200:
html.encoding = "utf8"
print(html.text)
else:
print("get error:" + html.url)
if __name__ == '__main__':
url = "http://httpbin.org/post"
post_html(url)