爬虫[02]

豆瓣电影TOP250数据爬取

完全是小白，不知道怎么弄。通过搜索发现了两个可以成功爬取的案例，在此仅作学习。

1.lxml requests 方法

来源：https://blog.csdn.net/m0_37788308/article/details/80378042

from lxml import etree
import requests
import time
# 通过循环爬取top250数据
for a in range(10):
    url = 'https://movie.douban.com/top250?start={}'.format(a*25)
    data = requests.get(url).text
    # print(data)
    s = etree.HTML(data)
    file = s.xpath('//*[@id="content"]/div/div[1]/ol/li')
    for div in file:
        movies_name = div.xpath('./div/div[2]/div[1]/a/span[1]/text()')[0]
        movies_score = div.xpath('./div/div[2]/div[2]/div/span[2]/text()')[0]
        movies_href = div.xpath('./div/div[2]/div[1]/a/@href')[0]
        movies_number = div.xpath('./div/div[2]/div[2]/div/span[4]/text()')[0].strip("(").strip( ).strip(")")
        movie_scrible = div.xpath('./div/div[2]/div[2]/p[2]/span/text()')
        # time.sleep(1)
        if len(movie_scrible)>0:
            print("{} {} {} {} {}".format(movies_name,movies_href,movies_score,movies_number,movie_scrible[0]))
        else:
            print("{} {} {} {}".format(movies_name,movies_href,movies_score,movies_number))

法二：https://blog.csdn.net/qq_41251963/article/details/81605331

import requests 
from lxml import html 
k = 1 
for i in range(10): 
    url = 'https://movie.douban.com/top250?start={}&filter='.format(i*25) 
    con = requests.get(url).content
    sel = html.fromstring(con) 

    # 所有的信息都在class属性为info的div标签里，可以先把这个节点取出来 //*[@id="content"]/div/div[1]/ol
    for i in sel.xpath('//div[@class="info"]'): 

        # 影片名称 
        title = i.xpath('div[@class="hd"]/a/span[@class="title"]/text()')[0] 
        #print(title)
        info = i.xpath('div[@class="bd"]/p[1]/text()') 
        # 导演演员信息 
        info_1 = info[0].replace(" ", "").replace("\n", "") 
        # 上映日期 
        date = info[1].replace(" ", "").replace("\n", "").split("/")[0] 
        # 制片国家 
        country = info[1].replace(" ", "").replace("\n", "").split("/")[1] 
        # 影片类型 
        geners = info[1].replace(" ", "").replace("\n", "").split("/")[2] 
        # 评分 
        rate = i.xpath('//span[@class="rating_num"]/text()')[0] 
        # 评论人数 
        comCount = i.xpath('//div[@class="star"]/span[4]/text()')[0] 

        # 打印结果看看 
        print ("TOP%s" % str(k))
        print( title, info_1, rate, date, country, geners, comCount )

        # 写入文件 
        with open("top250.txt", "a",encoding='utf-8') as f: 
            f.write("TOP%s\n影片名称：%s\n评分：%s %s\n上映日期：%s\n上映国家：%s\n%s\n" % (k, title, rate, comCount, date, country, info_1)) 

            f.write("==========================\n") 

        k += 1