爬虫：豆瓣电影 Top 250

本文链接：https://blog.csdn.net/weixin_64999250/article/details/133442832

本文介绍了使用Python的urllib和lxml库编写脚本，爬取豆瓣电影Top250的电影信息，包括导演、编剧、主演等，并将数据存储在.txt文件中。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

爬取豆瓣电影top250的每一部电影的导演，编剧，主演，上映时间，评分人数，剧情简介等信息，并存储在.txt文件中。


import urllib.request
from lxml import etree

def create_father_request(i):
    father_url = 'https://movie.douban.com/top250?start=' + str(i) + '&filter='
    print('第'+str(i/25+1)+'页:')
    print(father_url)
    headers={
        'User-Agent':'Mozilla/5.0(Windows NT 10.0;Win64;x64) AppleWebKit/537.36(KHTML,likeGecko)Chrome/117.0.0.0Safari/537.36'
    }
    request=urllib.request.Request(father_url,headers=headers)
    return  request
def get_content(request):
    response=urllib.request.urlopen(request)
    content=response.read().decode('utf-8')
    return content

def create_son_request (son_url):
    headers = {
        'User-Agent': 'Mozilla/5.0(Windows NT 10.0;Win64;x64) AppleWebKit/537.36(KHTML,likeGecko)Chrome/117.0.0.0Safari/537.36'
    }
    son_request = urllib.request.Request(son_url, headers=headers)
    return son_request


if __name__ == '__main__':

    for i in range(0,250,25):
        father_request=create_father_request(i)
        father_content=get_content(father_request)
        father_html_tree=etree.HTML(father_content)
        son_url_list=father_html_tree.xpath('//div[@class="hd"]/a/@href')
        for i in range(0,25):
            print(i+1)
            print(son_url_list[i])
            son_request=create_son_request(son_url_list[i])
            son_content=get_content(son_request)
            # print(son_content)
            son_html_tree=etree.HTML(son_content)
            name=son_html_tree.xpath('//span[@property="v:itemreviewed"]/text()')
            daoyan=son_html_tree.xpath('//*[@id="info"]/span[1]/span[2]/a/text()')
            bianju=son_html_tree.xpath('//*[@id="info"]/span[2]/span[2]/a/text()')
            zhuyan=son_html_tree.xpath('//*[@id="info"]/span[3]/span[2]//a//text()')
            leixing=son_html_tree.xpath('//*[@id="info"]/span[@property="v:genre"]/text()')
            shangyingdata=son_html_tree.xpath('//*[@id="info"]//span[@property="v:initialReleaseDate"]/text()')
            pianchang=son_html_tree.xpath('//*[@id="info"]//span[@property="v:runtime"]/text()')
            pingfenrenshu=son_html_tree.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/div/div[2]/a/span/text()')
            juqingjianjie=son_html_tree.xpath('//*[@id="link-report-intra"]//span[@property="v:summary"]/text()')
            # print(type(pianchang),pingfenrenshu,juqingjianjie)
            fp=open('data.txt','a',encoding='utf-8')

            fp.write('电影名:'+name[0]+'\n')

            fp.write('导演:')
            fp.write(daoyan[0]+'\n')

            fp.write('编剧:')
            for i in range(len(bianju)):
                fp.write(bianju[i] + ' ')
            fp.write('\n')

            fp.write('主演:')
            for i in range(len(zhuyan)):
                fp.write(zhuyan[i] + ' ')
            fp.write('\n')

            fp.write('类型:')
            for i in range(len(leixing)):
                fp.write(leixing[i] + ' ')
            fp.write('\n')

            fp.write('上映日期:')
            for i in range(len(shangyingdata)):
                fp.write(shangyingdata[i] + ' ')
            fp.write('\n')

            fp.write('片长:'+pianchang[0] + '\n')

            fp.write('评分人数:'+pingfenrenshu[0] + '\n')

            fp.write('剧情简介:' + juqingjianjie[0] + '\n')
    fp.close()