课时09 第二节课程：解析网页中的元素

find-all

from bs4 import BeautifulSoup
data=[]
path = r'D:\BaiduYunDownload\Python实战：：四周实现爬虫系统\课程资料\课程源码及作业参考答案\week1\1_2\1_2code_of_video\web\new_index.html'

with open(path, 'r') as f:
    Soup = BeautifulSoup(f.read(), 'lxml')#解析网页
    titles = Soup.select('body > div.main-content > ul > li > div.article-info > h3 > a')#从解析好的网页中选择元素，以列表形式返回
    images = Soup.select('body > div.main-content > ul > li > img')
    descs = Soup.select('body > div.main-content > ul > li > div.article-info > p.description')
    cates = Soup.select('body > div.main-content > ul > li > div.article-info > p.meta-info')
    rates = Soup.select('body > div.main-content > ul > li > div.rate > span')

for title, image, desc, cate, rate in zip(titles, images, descs, cates, rates):#遍历列表
    info = {
        'title':title.get_text(),
        'image':image.get('src'),
        'desc':desc.get_text(),
        'cate':list(cate.stripped_strings),
        'rate':rate.get_text()
    }
    data.append(info)#在data列表中增加字典元素

for i in data:
    if float(i['rate']) >= 4:
        print(i['title'], i['rate'])

from bs4 import BeautifulSoup
path = './1_2_homework_required/index.html'  #这里使用了相对路径,只要你本地有这个文件就能打开

with open(path, 'r') as wb_data: # 使用with open打开本地文件
    Soup = BeautifulSoup(wb_data, 'lxml') # 解析网页内容
    # print(wb_data)

    titles = Soup.select('body > div > div > div.col-md-9 > div > div > div > div.caption > h4 > a') # 复制每个元素的css selector 路径即可
    images = Soup.select('body > div > div > div.col-md-9 > div > div > div > img')
    reviews = Soup.select('body > div > div > div.col-md-9 > div > div > div > div.ratings > p.pull-right')
    prices = Soup.select('body > div > div > div.col-md-9 > div > div > div > div.caption > h4.pull-right')
    stars = Soup.select('body > div > div > div.col-md-9 > div > div > div > div.ratings > p:nth-of-type(2)')
    # 为了从父节点开始取,此处保留:nth-of-type(2),观察网页,多取几个星星的selector,就发现规律了
#body > div:nth-child(2) > div > div.col-md-9 > div:nth-child(2) > div:nth-child(1) > div > div.ratings > p:nth-child(2) > span:nth-child(2)，> span:nth-child(2)字段去掉，这个nth-child换成nth-of-type(2)
# print(titles,images,rates,prices,stars,sep='\n--------\n')  # 打印每个元素,其中sep='\n--------\n'是为了在不同元素之间添加分割线

for title, image, review, price, star in zip(titles, images, reviews, prices, stars):  # 使用for循环,把每个元素装到字典中
    data = {
        'title': title.get_text(), # 使用get_text()方法取出文本
        'image': image.get('src'), # 使用get 方法取出带有src的图片链接
        'review': review.get_text(),
        'price': price.get_text(),
        'star': len(star.find_all("span", class_='glyphicon glyphicon-star'))
        # 观察发现,每一个星星会有一次<span class="glyphicon glyphicon-star"></span>,所以我们统计有多少次,就知道有多少个星星了;
        # 使用find_all 统计有几处是★的样式,第一个参数定位标签名,第二个参数定位css 样式,具体可以参考BeautifulSoup 文档示例http://www.crummy.com/software/BeautifulSoup/bs4/doc.zh/#find-all;
        # 由于find_all()返回的结果是列表,我们再使用len()方法去计算列表中的元素个数,也就是星星的数量
    }
    print(data)

from bs4 import BeautifulSoup
import string

with open(r'D:\BaiduYunDownload\Python实战：：四周实现爬虫系统\课程资料\课程源码及作业参考答案\week1\1_2\1_2answer_of_homework\1_2_homework_required\index.html', 'r') as web_data:
    soup = BeautifulSoup(web_data, 'lxml')
    titles = soup.select(
        'body > div > div > div.col-md-9 > div > div > div > div.caption > h4 > a ')
    images = soup.select(
        'body > div > div > div.col-md-9 > div > div > div > img')
    reviews = soup.select(
        'body > div > div > div.col-md-9 > div > div > div > div.ratings > p.pull-right')
    prices = soup.select(
        'body > div > div > div.col-md-9 > div > div > div > div.caption > h4.pull-right')
    grades_crawler = soup.select(
        'body > div > div > div.col-md-9 > div > div > div > div.ratings > p:nth-of-type(2) > span ')
    # 上一行抓取所有的星星描述
    grades = []  # 设置一个空列表
    while len(grades_crawler) != 0:  # 循环条件长度不为0
        e = grades_crawler[0:5]  # 提取星星描述前五个元素，也就是一个商品的星级
        grades.insert(1, e)  # 把这五个商品星级的列表作为一个元素插入grades列表中
        del grades_crawler[0:5]  # 删除抓取到的描述列表的前五位

for title, image, review, price, grade in zip(titles, images, reviews, prices, grades):
    star = []
    b = str(grade)  # 字符串化列表
    c = b.replace('<span class="glyphicon glyphicon-star"></span>', '★')  # 将描述实五角星的替换为图案
    d = c.replace('<span class="glyphicon glyphicon-star-empty"></span>', '☆')  # 将描述虚五角星的替换为图案
    star.append(d)  # 将转化完的结果逐个插入列表star中
    data = {
        'title': title.get_text(),
        'image': image.get('src'),
        'review': review.get_text(),
        'price': price.get_text(),
        'grade': ''.join(star).replace('[', '').replace(']', '').replace(',', '').replace(' ', '')
    }
    print(data)