#python网络爬虫与信息提取
1.requests:自动爬取html页面自动网络请求提交
2.robot.txt:网络爬虫排除标准
3.projects实战
3.1.小规模:(requess)爬取网页,玩转网页
3.2.中规模:(scrapy)速度敏感 爬取网站,系列网站。
3.3.大规模(定制开发):爬取全网
#requests入门
#通用的爬虫框架
import requests
url = "https://item.jd.com/100002749549.html"
try:
r = requests.get(url)
r.raise_for_status()
r.encoding = r.apparent_encoding
print(r.text[:1000])
except:
print("爬取失败!")
#demo1.爬取亚马逊
import requests
r = requests.get("https://www.amazon.com/Deep-Learning-Adaptive-Computation-Machine/dp/0262035618/ref=sr_1_1?__mk_zh_CN=%E4%BA%9A%E9%A9%AC%E9%80%8A%E7%BD%91%E7%AB%99&keywords=Deep+learning&qid=1557108835&s=gateway&sr=8-1")
print(r.status_code)
print(r.encoding)
r.encoding = r.apparent_encoding
print(r.encoding)
print(r.text)
print(r.request.headers)
kv = {'user-agent':'Mozilla/5.0'}
url = "https://www.amazon.com/Deep-Learning-Adaptive-Computation-Machine/dp/0262035618/ref=sr_1_1?__mk_zh_CN=%E4%BA%9A%E9%A9%AC%E9%80%8A%E7%BD%91%E7%AB%99&keywords=Deep+learning&qid=1557108835&s=gateway&sr=8-1"
r = requests.get(url,headers = kv)
print(r.status_code)
print(r.request.headers)
print(r.text[:1000])
#d爬取亚马逊框架
import requests
url = "https://www.amazon.com/Deep-Learning-Adaptive-Computation-Machine/dp/0262035618/ref=sr_1_1?__mk_zh_CN=%E4%BA%9A%E" \
"9%A9%AC%E9%80%8A%E7%BD%91%E7%AB%99&keywords=Deep+learning&qid=1557108835&s=gateway&sr=8-1"
try:
kv = {'user-agent':'Mozilla/5.0'}
r = requests.get(url,headers = kv)
r.raise_for_status()
r.encoding = r.apparent_encoding
print(r.text[1000:2000])
except:
print("爬取失败!")
'''
#demo2.百度360搜索关键词提交
#2.1.百度
import requests
kv = {'wd':'python'}
r = requests.get("http://www.baidu.com/s",params = kv)
print(r.status_code)
print(r.request.url)
print(len(r.text))
#2.2 360
import requests
keyword = "python"
try:
kv = {'q':keyword}
r = requests.get("http://www.so.com/s",params = kv)
print(r.request.url)
r.raise_for_status()
print(len(r.text))
except:
print("爬取失败!")
#demo3.网络图片的爬取和存储
#格式:pictures.jpg
import requests
path = "E:\Pictures/xiaozhen.jpg"
url = "http://img0.dili360.com/ga/M00/00/7C/wKgBy1Q2a0OAMLwoAAizk5qejC0514.jpg@!rw9"
r= requests.get(url)
print(r.status_code)
with open(path,'wb') as f:
f.write(r.content)
f.close()
#demo4.图片爬取全代码
import requests
import os
url = "https://imgsa.baidu.com/forum/w%3D580/sign=e731858242ed2e73fce98624b701a16d/6c6e78310a55b3193a8fe8734ba98226cefc17eb.jpg"
root = "E:\Pictures"
path = root + url.split('/')[-1]
try:
if not os.path.exists(root):
os.mkdir(root)
if not os.path.exists(path):
r = requests.get(url)
with open(path,'wb') as f:
f.write(r.content)
f.close()
print("文件保存成功")
else:
print("文件已经存在")
except:
print("爬取失败")
#demo5.ip地址归属地查询
import requests
url = "http://www.ip138.com/ip.asp?ip="
r = requests.get(url+"202.204.80.112")
print(r.status_code)
print(r.text[-500:])
import requests
url = "http://m.ip138.com/ip.asp?ip="
try:
r = requests.get(url + '202.204.80.112')
r.raise_for_status()
r.encoding = r.apparent_encoding
print(r.text[-500:])
except:
print("爬取失败")