02步获取网页片段信息

最新推荐文章于 2024-06-20 01:30:12 发布

jidawanghao

最新推荐文章于 2024-06-20 01:30:12 发布

阅读量183

点赞数

本文链接：https://blog.csdn.net/jidawanghao/article/details/108375977

版权

from urllib.request import urlopen, Request
from lxml import etree
#构造函数，抓取第i页信息

    #  构造第i页的网址

url = 'https://movie.douban.com/top250?start=%s&filter='
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'}
ret = Request(url, headers=headers)
res = urlopen(ret)
html = res.read().decode('utf-8')
print(html)
html = etree.HTML(html)
datas = html.xpath('//ol[@class="grid_view"]/li')
a = 0
for data in datas:
    data_title = data.xpath('div/div[2]/div[@class="hd"]/a/span[1]/text()')
    data_info = data.xpath('div/div[2]/div[@class="bd"]/p[1]/text()')
    data_quote = data.xpath('div/div[2]/div[@class="bd"]/p[2]/span/text()')
    data_score = data.xpath('div/div[2]/div[@class="bd"]/div/span[@class="rating_num"]/text()')
    data_num = data.xpath('div/div[2]/div[@class="bd"]/div/span[4]/text()')
    data_picurl = data.xpath('div/div[1]/a/img/@src')
    print("No: " + str(1 * 25 + a + 1))
    print(data_title)
    # 保存电影信息到txt文件，下载封面图片
    with open('douban250.txt', 'a', encoding='utf-8')as f:
        # 封面图片保存路径和文件名
        picname = 'F:/top250/' + str(1 * 25 + a + 1) + '.jpg'
        f.write("No: " + str(1 * 25 + a + 1) + '\n')
        f.write(data_title[0] + '\n')
        f.write(str(data_info[0]).strip() + '\n')
        f.write(str(data_info[1]).strip() + '\n')
        # 因为发现有几部电影没有quote，所以这里加个判断，以免报错
        if data_quote:
            f.write(data_quote[0] + '\n')
        f.write(data_score[0] + '\n')
        f.write(data_num[0] + '\n')
        f.write('\n' * 3)
        # 下载封面图片到本地，路径为picname

    a += 1