首先,需要安装requests库
安装requests库
管理员方式运行cmd,输入pip install requests即可
1.亚马逊网站的爬取实例(对来源进行了审查)
#亚马逊会检测来源
import requests
url="https://www.amazon.cn/gp/product/B01M8L5Z3Y"
try:
kv={'user-agent':'Mozilla/5.0'}
r=requests.get(url,headers=kv)
r.raise_for_status()
r.encoding=r.apparent_encoding
print(r.text[1000:2000])
except:
print("抓取失败")
2.百度搜索
import requests
keyword="Python"
try:
kv={'wd':keyword}
r=requests.get("http://www.baidu.com/s",params=kv)
r.raise_for_status()
print(len(r.text))
except:
print("爬取失败")
3.ip地址的查询
import requests
url="http://m.ip138.com/ip.asp?ip="
try:
r=requests.get(url+'222.204.3.221')
r.raise_for_status()
r.encoding=r.apparent_encoding
print(r.text[-500:])
except:
print("爬取失败")
4.图片/视频的爬取
#图片的爬取
import requests
import os
url="https://c-ssl.duitang.com/uploads/item/201803/24/20180324081023_8FVre.thumb.700_0.jpeg"
root="E://pic//"
path=root+url.split('/')[-1]
try:
if not os.path.exists(root):
os.mkdir(root)
if not os.path.exists(path):
r=requests.get(url)
with open(path,'wb') as f:
f.write(r.content)
f.close()
print("文件保存成功")
else:
print("文件已存在")
except:
print("爬取失败")