爬虫实战-豆瓣电影Top250

爬豆瓣电影Top250电影信息，电影简介，热门评论。。。

from lxml import etree
import requests,re,time


class douBan():
    def __init__(self):
        self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36'}
        self.url='https://movie.douban.com/top250?start={}&filter='

    def get_url_list(self):
        return [self.url.format(str(i)) for i in range(0, 250, 25)]

    def get_movie_url(self,url):
        #print(url)
        html = requests.get(url, headers=self.headers)
        return html.text

    def get_movie_info(self,html):
        selector = etree.HTML(html)
        movies = selector.xpath("//div[@class='info']")
        #content_list = []
        for each in movies:
            item={}
            # 标题
            item['title'] = each.xpath(".//span[@class='title'][1]/text()")[0]
            # 信息
            item['director'] = each.xpath(".//div[@class='bd']/p/text()")[0].strip().replace("\xa0"," ")
            # 评分
            item['star'] = each.xpath(".//div[@class='star']/span[@class='rating_num']/text()")[0]
            # 简介
            item['quote']= each.xpath(".//p[@class='quote']/span/text()")[0]
            item['quote']=item['quote']if len(item["quote"]) > 0 else None

            #content_list.append(item)
            print(item)

        #评论：待添加
        movie_hrefs = selector.xpath('//div[@class="hd"]/a/@href')
        for movie_href in movie_hrefs:
            html=self.get_movie_url(movie_href)
            selector = etree.HTML(html)
            






            # movie_name = selector.xpath('//div[@id="content"]/h1/span[1]/text()')[0]
            # director = selector.xpath('//*[@id="info"]/span[1]/span[2]/a/text()')[0]
            # actors = selector.xpath('//span[@class="actor"]/span[2]')[0]
            # actor = actors.xpath('string(.)')
            # style = re.findall('<span property="v:genre">(.*?)</span>', html.text, re.S)[0]
            # country = re.findall('制片国家/地区:</span>(.*?)<br/>', html.text, re.S)[0]
            # release_time = re.findall('上映日期:</span>.*?>(.*?)</span>', html.text, re.S)[0]
            # time = re.findall('片长:</span>.*?>(.*?)</span>', html.text, re.S)[0]
            # score = selector.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()')[0]
            #


    def run(self): #实现主要逻辑
        #1.url_list
        url_list = self.get_url_list()
        #2.遍历，发送请求，获取响应
        for url in url_list:
            html_str = self.get_movie_url(url)
            #3.提取数据
            time.sleep(2)
            content_list = self.get_movie_info(html_str)
            #4.保存
            #self.save_content_list(content_list)


if __name__ == '__main__':
    douban=douBan()
    douban.run()