1 前言
本文列举了几个requests库的爬虫小程序,内容来自北京理工大学嵩天老师的视频教程,大家想入门学习爬虫的可以搜一下学习。点击进入Requests库官方教程。
2 Requests库使用实例
2.1 京东商品页面的爬取
import requests
# 通用代码框架
def getHTMLText(url):
try:
r = requests.get(url, timeout=30)
r.raise_for_status # 如果返回的不是200,则抛异常
r.encoding = r.apparent_encoding # 将备选编码方式赋值给编码方式,功能是针对中文网页使用UTF-8编码
return r.text[:500] # text内容可能过多,因此只显示前500字符
except:
return "产生异常"
if __name__ == "__main__":
url = "https://item.jd.com/100010823452.html"
print(getHTMLText(url))
2.2 亚马逊商品页面的爬取
import requests
# 通用代码框架
def getHTMLText(url):
try:
# 将user-agent从python-requests/2.22.0更改为Mozilla/5.0
# 由于亚马逊网站有一定的反爬虫机制,上述更改可以绕过部分的机制,状态码会从503变为200
kv = {'user-agent':'Mozilla/5.0'}
r = requests.get(url, headers = kv)
r.raise_for_status
r.encoding = r.apparent_encoding
return r.text[-500:]
except:
return "产生异常"
if __name__ == "__main__":
url = "https://www.amazon.cn/dp/B00CWPIDHI?_encoding=UTF8&ref_=nav_kdpcwin"
print(getHTMLText(url))
2.3 百度、360搜索关键词(百度目前这个api返回的是需要验证码,失效了)
import requests
# 通用代码框架
def getHTMLText(url):
try:
kv = {'wd':'Python'}
r = requests.get(url, params = kv)
r.raise_for_status
r.encoding = r.apparent_encoding
return r.text[-500:]
except:
return "产生异常"
if __name__ == "__main__":
url = "http://www.baidu.com/s"
print(getHTMLText(url))
2.4 网络图片的爬取与存储
import requests
import os
# 网页图片后缀名命名的图片,爬取下载到本地
def getHTMLText(url):
try:
if not os.path.exists(root):
os.mkdir(root)
if not os.path.exists(path):
r = requests.get(url)
# 以二进制形式写入
with open(path, 'wb') as f:
f.write(r.content)
f.close
print("图片保存成功")
else:
print("图片已经存在")
except:
return "产生异常"
if __name__ == "__main__":
url = "http://image.ngchina.com.cn/2020/0221/20200221013711877.jpg"
root = "C:/Users/Desktop/Python_爬虫/test/"
path = root + url.split('/')[-1]
print(getHTMLText(url))
2.5 IP地址归属地自动查询
import requests
def getHTMLText(url):
try:
kv = {'user-agent':'Mozilla/5.0'}
r = requests.get(url + '55.55.55.55', headers = kv)
r.raise_for_status
r.encoding = r.apparent_encoding
return r.text
except:
return "产生异常"
if __name__ == "__main__":
url = "http://www.ip138.com/ips138.asp?ip="
print(getHTMLText(url))