使用BeautifulSoup写爬虫,需要完成以下三个步骤。
0 使用BeautifulSoup解析一个网页
1 找到所有你需要的标签
2 从标签中提取需要的数据
xpath 与 css selector 有点象现实生活的地址,感觉也有点像文字检索。
# coding:utf-8
from bs4 import BeautifulSoup
with open("./index.html",'r') as web_file:
soup = BeautifulSoup(web_file, 'lxml')
imgs = soup.select("body > div > div > div > div > div > div > img")
names = soup.select("body > div > div > div > div > div > div > div > h4 > a")
prices = soup.select("body > div > div > div > div > div > div > div > h4.pull-right")
reviews = soup.select("body > div > div > div > div > div > div > div > h4.pull-right")
stars = soup.select("body > div > div > div.col-md-9 > div > div > div > div.ratings > p:nth-of-type(2)")
for img, name, price, reviews, star in zip(imgs, names, prices, reviews, stars):
data = {
'image':img.get('src'),
'name':name.get_text(),
'price':price.get_text(),
'reviews':reviews.get_text(),
'star':len(star.find_all('span','glyphicon-star'))
}
print(data)