python捕捉数据

doudan数据捕获

import requests
from lxml import etree
from openpyxl import Workbook
#etree 模块提供了一组功能，用于解析、构建和操作XML和HTML文档，提供了比标准库更高效的XML处理方法。
def get_douban_movies(url):
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.0.0'
    }
    response = requests.get(url,headers=headers)
    # print(response.text)
    # etree.HTML(response.text)#括号内传递参数，传递我们需要解析的文本
    html = etree.HTML(response.text)
    ranks = html.xpath('//div[@class="item"]//em/text()')
    ranks = [int(rank) for rank in ranks]#将字符串转化成整数
    names = html.xpath('//div[@class="item"]//span[@class="title"][1]/text()')
    gradeds = html.xpath('//div[@class="item"]//span[@class="rating_num"]/text()')
    gradeds = [float(graded) for graded in gradeds]#将字符串转化成整数
    sentenses = html.xpath('//div[@class="item"]//p//span/text()')

    movies = []
    for rank, name, graded, sentense in zip(ranks,names,gradeds,sentenses):
        movies.append((rank,name,graded,sentense))
    return movies

def save_to_excel(movies):
    wb = Workbook()
    # 激活
    ws =wb.active
    ws.append(['排名','电影名称','评分','经典语句'])#加入表头
    for movie in movies:
        ws.append(movie)
    wb.save("豆瓣电影250数据爬取表.xlsx")
    wb.close()
    #关闭网站

if __name__ == '__main__':
    base_url = "https://movie.douban.com/top250"
    data = []
    # 前面2个是范围，后面一个是数每次增加25
    for i in range(0,251,25):
        url = f"{base_url}?start={i}"
        movies = get_douban_movies(url)
        data += movies
    print(data)
    save_to_excel(data)